From 0e9236475b6bd55683d78bd59750ef7d3f1fb918 Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Thu, 7 Jan 2016 20:23:38 +0100 Subject: [PATCH] Update s3 library (again) --- Godeps/Godeps.json | 4 +- .../src/github.com/minio/minio-go/.travis.yml | 2 +- .../src/github.com/minio/minio-go/README.md | 10 +- .../minio/minio-go/api-definitions.go | 57 +- .../minio/minio-go/api-error-response.go | 93 ++- ...-fget-object.go => api-get-object-file.go} | 2 +- .../src/github.com/minio/minio-go/api-get.go | 343 +++++--- .../src/github.com/minio/minio-go/api-list.go | 109 ++- .../minio/minio-go/api-presigned.go | 11 +- .../minio/minio-go/api-put-bucket.go | 4 +- ...-fput-object.go => api-put-object-file.go} | 227 +++--- .../minio-go/api-put-object-multipart.go | 421 ++++++++++ .../minio/minio-go/api-put-object-partial.go | 378 --------- .../minio/minio-go/api-put-object-readat.go | 196 +++++ .../minio/minio-go/api-put-object.go | 529 +++--------- .../github.com/minio/minio-go/api-remove.go | 51 +- .../minio/minio-go/api-s3-definitions.go | 66 +- .../src/github.com/minio/minio-go/api-stat.go | 25 +- .../src/github.com/minio/minio-go/api.go | 129 ++- .../minio/minio-go/api_functional_v2_test.go | 751 ++++++++++++++++++ ...onal_test.go => api_functional_v4_test.go} | 340 ++++++-- .../{api_private_test.go => api_unit_test.go} | 117 ++- .../github.com/minio/minio-go/appveyor.yml | 4 +- .../github.com/minio/minio-go/bucket-acl.go | 14 +- .../github.com/minio/minio-go/bucket-cache.go | 31 +- .../minio/minio-go/common-methods.go | 52 -- .../github.com/minio/minio-go/constants.go | 12 +- .../minio/minio-go/examples/play/getobject.go | 26 +- .../examples/play/getobjectpartial.go | 91 --- .../minio/minio-go/examples/play/putobject.go | 3 +- .../examples/play/putobjectpartial.go | 56 -- .../minio/minio-go/examples/s3/getobject.go | 14 +- .../minio-go/examples/s3/getobjectpartial.go | 92 --- .../minio/minio-go/examples/s3/putobject.go | 3 +- .../minio-go/examples/s3/putobjectpartial.go | 57 -- .../github.com/minio/minio-go/post-policy.go | 57 +- .../minio/minio-go/request-signature-v2.go | 40 +- .../minio/minio-go/request-signature-v4.go | 87 +- .../minio/minio-go/signature-type.go | 16 + .../src/github.com/minio/minio-go/tempfile.go | 2 +- .../src/github.com/minio/minio-go/utils.go | 24 +- 41 files changed, 2793 insertions(+), 1753 deletions(-) rename Godeps/_workspace/src/github.com/minio/minio-go/{api-fget-object.go => api-get-object-file.go} (97%) rename Godeps/_workspace/src/github.com/minio/minio-go/{api-fput-object.go => api-put-object-file.go} (67%) create mode 100644 Godeps/_workspace/src/github.com/minio/minio-go/api-put-object-multipart.go delete mode 100644 Godeps/_workspace/src/github.com/minio/minio-go/api-put-object-partial.go create mode 100644 Godeps/_workspace/src/github.com/minio/minio-go/api-put-object-readat.go create mode 100644 Godeps/_workspace/src/github.com/minio/minio-go/api_functional_v2_test.go rename Godeps/_workspace/src/github.com/minio/minio-go/{api_functional_test.go => api_functional_v4_test.go} (62%) rename Godeps/_workspace/src/github.com/minio/minio-go/{api_private_test.go => api_unit_test.go} (60%) delete mode 100644 Godeps/_workspace/src/github.com/minio/minio-go/common-methods.go delete mode 100644 Godeps/_workspace/src/github.com/minio/minio-go/examples/play/getobjectpartial.go delete mode 100644 Godeps/_workspace/src/github.com/minio/minio-go/examples/play/putobjectpartial.go delete mode 100644 Godeps/_workspace/src/github.com/minio/minio-go/examples/s3/getobjectpartial.go delete mode 100644 Godeps/_workspace/src/github.com/minio/minio-go/examples/s3/putobjectpartial.go diff --git a/Godeps/Godeps.json b/Godeps/Godeps.json index ec42f905f..03b9fcb05 100644 --- a/Godeps/Godeps.json +++ b/Godeps/Godeps.json @@ -24,8 +24,8 @@ }, { "ImportPath": "github.com/minio/minio-go", - "Comment": "v0.2.5-201-g410319e", - "Rev": "410319e0e39a372998f4d9cd2b9da4ff243ae388" + "Comment": "v0.2.5-205-g38be406", + "Rev": "38be40605dc37d2d7ec06169218365b46ae33e4b" }, { "ImportPath": "github.com/pkg/sftp", diff --git a/Godeps/_workspace/src/github.com/minio/minio-go/.travis.yml b/Godeps/_workspace/src/github.com/minio/minio-go/.travis.yml index 96a4e2a47..f76844876 100644 --- a/Godeps/_workspace/src/github.com/minio/minio-go/.travis.yml +++ b/Godeps/_workspace/src/github.com/minio/minio-go/.travis.yml @@ -15,7 +15,7 @@ go: script: - go vet ./... -- go test -test.short -race -v ./... +- go test -short -race -v ./... notifications: slack: diff --git a/Godeps/_workspace/src/github.com/minio/minio-go/README.md b/Godeps/_workspace/src/github.com/minio/minio-go/README.md index 5417d8f14..e32bf6f5f 100644 --- a/Godeps/_workspace/src/github.com/minio/minio-go/README.md +++ b/Godeps/_workspace/src/github.com/minio/minio-go/README.md @@ -67,14 +67,14 @@ func main() { * [RemoveBucket(bucketName) error](examples/s3/removebucket.go) * [GetBucketACL(bucketName) (BucketACL, error)](examples/s3/getbucketacl.go) * [SetBucketACL(bucketName, BucketACL) error)](examples/s3/setbucketacl.go) -* [ListBuckets() []BucketStat](examples/s3/listbuckets.go) -* [ListObjects(bucketName, objectPrefix, recursive, chan<- struct{}) <-chan ObjectStat](examples/s3/listobjects.go) -* [ListIncompleteUploads(bucketName, prefix, recursive, chan<- struct{}) <-chan ObjectMultipartStat](examples/s3/listincompleteuploads.go) +* [ListBuckets() []BucketInfo](examples/s3/listbuckets.go) +* [ListObjects(bucketName, objectPrefix, recursive, chan<- struct{}) <-chan ObjectInfo](examples/s3/listobjects.go) +* [ListIncompleteUploads(bucketName, prefix, recursive, chan<- struct{}) <-chan ObjectMultipartInfo](examples/s3/listincompleteuploads.go) ### Object Operations. * [PutObject(bucketName, objectName, io.Reader, size, contentType) error](examples/s3/putobject.go) -* [GetObject(bucketName, objectName) (io.ReadCloser, ObjectStat, error)](examples/s3/getobject.go) -* [StatObject(bucketName, objectName) (ObjectStat, error)](examples/s3/statobject.go) +* [GetObject(bucketName, objectName) (io.ReadCloser, ObjectInfo, error)](examples/s3/getobject.go) +* [StatObject(bucketName, objectName) (ObjectInfo, error)](examples/s3/statobject.go) * [RemoveObject(bucketName, objectName) error](examples/s3/removeobject.go) * [RemoveIncompleteUpload(bucketName, objectName) <-chan error](examples/s3/removeincompleteupload.go) diff --git a/Godeps/_workspace/src/github.com/minio/minio-go/api-definitions.go b/Godeps/_workspace/src/github.com/minio/minio-go/api-definitions.go index 123de1850..fd0a613a8 100644 --- a/Godeps/_workspace/src/github.com/minio/minio-go/api-definitions.go +++ b/Godeps/_workspace/src/github.com/minio/minio-go/api-definitions.go @@ -16,26 +16,27 @@ package minio -import ( - "io" - "time" -) +import "time" -// BucketStat container for bucket metadata. -type BucketStat struct { +// BucketInfo container for bucket metadata. +type BucketInfo struct { // The name of the bucket. Name string // Date the bucket was created. CreationDate time.Time } -// ObjectStat container for object metadata. -type ObjectStat struct { - ETag string - Key string - LastModified time.Time - Size int64 - ContentType string +// ObjectInfo container for object metadata. +type ObjectInfo struct { + // An ETag is optionally set to md5sum of an object. In case of multipart objects, + // ETag is of the form MD5SUM-N where MD5SUM is md5sum of all individual md5sums of + // each parts concatenated into one string. + ETag string + + Key string // Name of the object + LastModified time.Time // Date and time the object was last modified. + Size int64 // Size in bytes of the object. + ContentType string // A standard MIME type describing the format of the object data. // Owner name. Owner struct { @@ -50,18 +51,21 @@ type ObjectStat struct { Err error } -// ObjectMultipartStat container for multipart object metadata. -type ObjectMultipartStat struct { +// ObjectMultipartInfo container for multipart object metadata. +type ObjectMultipartInfo struct { // Date and time at which the multipart upload was initiated. Initiated time.Time `type:"timestamp" timestampFormat:"iso8601"` Initiator initiator Owner owner + // The type of storage to use for the object. Defaults to 'STANDARD'. StorageClass string // Key of the object for which the multipart upload was initiated. - Key string + Key string + + // Size in bytes of the object. Size int64 // Upload ID that identifies the multipart upload. @@ -70,24 +74,3 @@ type ObjectMultipartStat struct { // Error Err error } - -// partData - container for each part. -type partData struct { - MD5Sum []byte - Sha256Sum []byte - ReadCloser io.ReadCloser - Size int64 - Number int // partData number. - - // Error - Err error -} - -// putObjectData - container for each single PUT operation. -type putObjectData struct { - MD5Sum []byte - Sha256Sum []byte - ReadCloser io.ReadCloser - Size int64 - ContentType string -} diff --git a/Godeps/_workspace/src/github.com/minio/minio-go/api-error-response.go b/Godeps/_workspace/src/github.com/minio/minio-go/api-error-response.go index 4d7e30fc1..ca15164f9 100644 --- a/Godeps/_workspace/src/github.com/minio/minio-go/api-error-response.go +++ b/Godeps/_workspace/src/github.com/minio/minio-go/api-error-response.go @@ -17,7 +17,6 @@ package minio import ( - "encoding/json" "encoding/xml" "fmt" "net/http" @@ -36,7 +35,7 @@ import ( */ -// ErrorResponse is the type error returned by some API operations. +// ErrorResponse - Is the typed error returned by all API operations. type ErrorResponse struct { XMLName xml.Name `xml:"Error" json:"-"` Code string @@ -46,12 +45,13 @@ type ErrorResponse struct { RequestID string `xml:"RequestId"` HostID string `xml:"HostId"` - // This is a new undocumented field, set only if available. + // Region where the bucket is located. This header is returned + // only in HEAD bucket and ListObjects response. AmzBucketRegion string } -// ToErrorResponse returns parsed ErrorResponse struct, if input is nil or not ErrorResponse return value is nil -// this fuction is useful when some one wants to dig deeper into the error structures over the network. +// ToErrorResponse - Returns parsed ErrorResponse struct from body and +// http headers. // // For example: // @@ -61,7 +61,6 @@ type ErrorResponse struct { // reader, stat, err := s3.GetObject(...) // if err != nil { // resp := s3.ToErrorResponse(err) -// fmt.Println(resp.ToXML()) // } // ... func ToErrorResponse(err error) ErrorResponse { @@ -73,47 +72,32 @@ func ToErrorResponse(err error) ErrorResponse { } } -// ToXML send raw xml marshalled as string -func (e ErrorResponse) ToXML() string { - b, err := xml.Marshal(&e) - if err != nil { - panic(err) - } - return string(b) -} - -// ToJSON send raw json marshalled as string -func (e ErrorResponse) ToJSON() string { - b, err := json.Marshal(&e) - if err != nil { - panic(err) - } - return string(b) -} - -// Error formats HTTP error string +// Error - Returns HTTP error string func (e ErrorResponse) Error() string { return e.Message } -// Common reporting string +// Common string for errors to report issue location in unexpected +// cases. const ( reportIssue = "Please report this issue at https://github.com/minio/minio-go/issues." ) -// HTTPRespToErrorResponse returns a new encoded ErrorResponse structure +// HTTPRespToErrorResponse returns a new encoded ErrorResponse +// structure as error. func HTTPRespToErrorResponse(resp *http.Response, bucketName, objectName string) error { if resp == nil { msg := "Response is empty. " + reportIssue return ErrInvalidArgument(msg) } - var errorResponse ErrorResponse - err := xmlDecoder(resp.Body, &errorResponse) + var errResp ErrorResponse + err := xmlDecoder(resp.Body, &errResp) + // Xml decoding failed with no body, fall back to HTTP headers. if err != nil { switch resp.StatusCode { case http.StatusNotFound: if objectName == "" { - errorResponse = ErrorResponse{ + errResp = ErrorResponse{ Code: "NoSuchBucket", Message: "The specified bucket does not exist.", BucketName: bucketName, @@ -122,7 +106,7 @@ func HTTPRespToErrorResponse(resp *http.Response, bucketName, objectName string) AmzBucketRegion: resp.Header.Get("x-amz-bucket-region"), } } else { - errorResponse = ErrorResponse{ + errResp = ErrorResponse{ Code: "NoSuchKey", Message: "The specified key does not exist.", BucketName: bucketName, @@ -133,7 +117,7 @@ func HTTPRespToErrorResponse(resp *http.Response, bucketName, objectName string) } } case http.StatusForbidden: - errorResponse = ErrorResponse{ + errResp = ErrorResponse{ Code: "AccessDenied", Message: "Access Denied.", BucketName: bucketName, @@ -143,7 +127,7 @@ func HTTPRespToErrorResponse(resp *http.Response, bucketName, objectName string) AmzBucketRegion: resp.Header.Get("x-amz-bucket-region"), } case http.StatusConflict: - errorResponse = ErrorResponse{ + errResp = ErrorResponse{ Code: "Conflict", Message: "Bucket not empty.", BucketName: bucketName, @@ -152,7 +136,7 @@ func HTTPRespToErrorResponse(resp *http.Response, bucketName, objectName string) AmzBucketRegion: resp.Header.Get("x-amz-bucket-region"), } default: - errorResponse = ErrorResponse{ + errResp = ErrorResponse{ Code: resp.Status, Message: resp.Status, BucketName: bucketName, @@ -162,10 +146,21 @@ func HTTPRespToErrorResponse(resp *http.Response, bucketName, objectName string) } } } - return errorResponse + + // AccessDenied without a signature mismatch code, usually means + // that the bucket policy has certain restrictions where some API + // operations are not allowed. Handle this case so that top level + // callers can interpret this easily and fall back if needed to a + // lower functionality call. Read each individual API specific + // code for such fallbacks. + if errResp.Code == "AccessDenied" && errResp.Message == "Access Denied" { + errResp.Code = "NotImplemented" + errResp.Message = "Operation is not allowed according to your bucket policy." + } + return errResp } -// ErrEntityTooLarge input size is larger than supported maximum. +// ErrEntityTooLarge - Input size is larger than supported maximum. func ErrEntityTooLarge(totalSize int64, bucketName, objectName string) error { msg := fmt.Sprintf("Your proposed upload size ‘%d’ exceeds the maximum allowed object size '5GiB' for single PUT operation.", totalSize) return ErrorResponse{ @@ -176,7 +171,19 @@ func ErrEntityTooLarge(totalSize int64, bucketName, objectName string) error { } } -// ErrUnexpectedShortRead unexpected shorter read of input buffer from target. +// ErrEntityTooSmall - Input size is smaller than supported minimum. +func ErrEntityTooSmall(totalSize int64, bucketName, objectName string) error { + msg := fmt.Sprintf("Your proposed upload size ‘%d’ is below the minimum allowed object size '0B' for single PUT operation.", totalSize) + return ErrorResponse{ + Code: "EntityTooLarge", + Message: msg, + BucketName: bucketName, + Key: objectName, + } +} + +// ErrUnexpectedShortRead - Unexpected shorter read of input buffer from +// target. func ErrUnexpectedShortRead(totalRead, totalSize int64, bucketName, objectName string) error { msg := fmt.Sprintf("Data read ‘%s’ is shorter than the size ‘%s’ of input buffer.", strconv.FormatInt(totalRead, 10), strconv.FormatInt(totalSize, 10)) @@ -188,7 +195,7 @@ func ErrUnexpectedShortRead(totalRead, totalSize int64, bucketName, objectName s } } -// ErrUnexpectedEOF unexpected end of file reached. +// ErrUnexpectedEOF - Unexpected end of file reached. func ErrUnexpectedEOF(totalRead, totalSize int64, bucketName, objectName string) error { msg := fmt.Sprintf("Data read ‘%s’ is not equal to the size ‘%s’ of the input Reader.", strconv.FormatInt(totalRead, 10), strconv.FormatInt(totalSize, 10)) @@ -200,7 +207,7 @@ func ErrUnexpectedEOF(totalRead, totalSize int64, bucketName, objectName string) } } -// ErrInvalidBucketName - invalid bucket name response. +// ErrInvalidBucketName - Invalid bucket name response. func ErrInvalidBucketName(message string) error { return ErrorResponse{ Code: "InvalidBucketName", @@ -209,7 +216,7 @@ func ErrInvalidBucketName(message string) error { } } -// ErrInvalidObjectName - invalid object name response. +// ErrInvalidObjectName - Invalid object name response. func ErrInvalidObjectName(message string) error { return ErrorResponse{ Code: "NoSuchKey", @@ -218,7 +225,7 @@ func ErrInvalidObjectName(message string) error { } } -// ErrInvalidParts - invalid number of parts. +// ErrInvalidParts - Invalid number of parts. func ErrInvalidParts(expectedParts, uploadedParts int) error { msg := fmt.Sprintf("Unexpected number of parts found Want %d, Got %d", expectedParts, uploadedParts) return ErrorResponse{ @@ -228,11 +235,11 @@ func ErrInvalidParts(expectedParts, uploadedParts int) error { } } -// ErrInvalidObjectPrefix - invalid object prefix response is +// ErrInvalidObjectPrefix - Invalid object prefix response is // similar to object name response. var ErrInvalidObjectPrefix = ErrInvalidObjectName -// ErrInvalidArgument - invalid argument response. +// ErrInvalidArgument - Invalid argument response. func ErrInvalidArgument(message string) error { return ErrorResponse{ Code: "InvalidArgument", diff --git a/Godeps/_workspace/src/github.com/minio/minio-go/api-fget-object.go b/Godeps/_workspace/src/github.com/minio/minio-go/api-get-object-file.go similarity index 97% rename from Godeps/_workspace/src/github.com/minio/minio-go/api-fget-object.go rename to Godeps/_workspace/src/github.com/minio/minio-go/api-get-object-file.go index ee96a6cb9..b73ed2cb3 100644 --- a/Godeps/_workspace/src/github.com/minio/minio-go/api-fget-object.go +++ b/Godeps/_workspace/src/github.com/minio/minio-go/api-get-object-file.go @@ -22,7 +22,7 @@ import ( "path/filepath" ) -// FGetObject - get object to a file. +// FGetObject - download contents of an object to a local file. func (c Client) FGetObject(bucketName, objectName, filePath string) error { // Input validation. if err := isValidBucketName(bucketName); err != nil { diff --git a/Godeps/_workspace/src/github.com/minio/minio-go/api-get.go b/Godeps/_workspace/src/github.com/minio/minio-go/api-get.go index 7596278af..46643a5c7 100644 --- a/Godeps/_workspace/src/github.com/minio/minio-go/api-get.go +++ b/Godeps/_workspace/src/github.com/minio/minio-go/api-get.go @@ -28,15 +28,18 @@ import ( "time" ) -// GetBucketACL get the permissions on an existing bucket. +// GetBucketACL - Get the permissions on an existing bucket. // // Returned values are: // -// private - owner gets full access. -// public-read - owner gets full access, others get read access. -// public-read-write - owner gets full access, others get full access too. -// authenticated-read - owner gets full access, authenticated users get read access. +// private - Owner gets full access. +// public-read - Owner gets full access, others get read access. +// public-read-write - Owner gets full access, others get full access +// too. +// authenticated-read - Owner gets full access, authenticated users +// get read access. func (c Client) GetBucketACL(bucketName string) (BucketACL, error) { + // Input validation. if err := isValidBucketName(bucketName); err != nil { return "", err } @@ -73,9 +76,10 @@ func (c Client) GetBucketACL(bucketName string) (BucketACL, error) { return "", err } - // We need to avoid following de-serialization check for Google Cloud Storage. - // On Google Cloud Storage "private" canned ACL's policy do not have grant list. - // Treat it as a valid case, check for all other vendors. + // We need to avoid following de-serialization check for Google + // Cloud Storage. On Google Cloud Storage "private" canned ACL's + // policy do not have grant list. Treat it as a valid case, check + // for all other vendors. if !isGoogleEndpoint(c.endpointURL) { if policy.AccessControlList.Grant == nil { errorResponse := ErrorResponse{ @@ -90,8 +94,8 @@ func (c Client) GetBucketACL(bucketName string) (BucketACL, error) { } } - // boolean cues to indentify right canned acls. - var publicRead, publicWrite bool + // Boolean cues to indentify right canned acls. + var publicRead, publicWrite, authenticatedRead bool // Handle grants. grants := policy.AccessControlList.Grant @@ -100,7 +104,8 @@ func (c Client) GetBucketACL(bucketName string) (BucketACL, error) { continue } if g.Grantee.URI == "http://acs.amazonaws.com/groups/global/AuthenticatedUsers" && g.Permission == "READ" { - return BucketACL("authenticated-read"), nil + authenticatedRead = true + break } else if g.Grantee.URI == "http://acs.amazonaws.com/groups/global/AllUsers" && g.Permission == "WRITE" { publicWrite = true } else if g.Grantee.URI == "http://acs.amazonaws.com/groups/global/AllUsers" && g.Permission == "READ" { @@ -108,15 +113,19 @@ func (c Client) GetBucketACL(bucketName string) (BucketACL, error) { } } - // public write and not enabled. return. + // Verify if acl is authenticated read. + if authenticatedRead { + return BucketACL("authenticated-read"), nil + } + // Verify if acl is private. if !publicWrite && !publicRead { return BucketACL("private"), nil } - // public write not enabled but public read is. return. + // Verify if acl is public-read. if !publicWrite && publicRead { return BucketACL("public-read"), nil } - // public read and public write are enabled return. + // Verify if acl is public-read-write. if publicRead && publicWrite { return BucketACL("public-read-write"), nil } @@ -129,47 +138,30 @@ func (c Client) GetBucketACL(bucketName string) (BucketACL, error) { } } -// GetObject gets object content from specified bucket. -// You may also look at GetPartialObject. -func (c Client) GetObject(bucketName, objectName string) (io.ReadCloser, ObjectStat, error) { +// GetObject - returns an seekable, readable object. +func (c Client) GetObject(bucketName, objectName string) (*Object, error) { + // Input validation. if err := isValidBucketName(bucketName); err != nil { - return nil, ObjectStat{}, err + return nil, err } if err := isValidObjectName(objectName); err != nil { - return nil, ObjectStat{}, err + return nil, err } - // get the whole object as a stream, no seek or resume supported for this. - return c.getObject(bucketName, objectName, 0, 0) -} - -// ReadAtCloser readat closer interface. -type ReadAtCloser interface { - io.ReaderAt - io.Closer -} - -// GetObjectPartial returns a io.ReadAt for reading sparse entries. -func (c Client) GetObjectPartial(bucketName, objectName string) (ReadAtCloser, ObjectStat, error) { - if err := isValidBucketName(bucketName); err != nil { - return nil, ObjectStat{}, err - } - if err := isValidObjectName(objectName); err != nil { - return nil, ObjectStat{}, err - } - // Send an explicit stat to get the actual object size. - objectStat, err := c.StatObject(bucketName, objectName) + // Send an explicit info to get the actual object size. + objectInfo, err := c.StatObject(bucketName, objectName) if err != nil { - return nil, ObjectStat{}, err + return nil, err } // Create request channel. - reqCh := make(chan readAtRequest) + reqCh := make(chan readRequest) // Create response channel. - resCh := make(chan readAtResponse) + resCh := make(chan readResponse) // Create done channel. doneCh := make(chan struct{}) - // This routine feeds partial object data as and when the caller reads. + // This routine feeds partial object data as and when the caller + // reads. go func() { defer close(reqCh) defer close(resCh) @@ -185,21 +177,21 @@ func (c Client) GetObjectPartial(bucketName, objectName string) (ReadAtCloser, O // Get shortest length. // NOTE: Last remaining bytes are usually smaller than // req.Buffer size. Use that as the final length. - length := math.Min(float64(len(req.Buffer)), float64(objectStat.Size-req.Offset)) + length := math.Min(float64(len(req.Buffer)), float64(objectInfo.Size-req.Offset)) httpReader, _, err := c.getObject(bucketName, objectName, req.Offset, int64(length)) if err != nil { - resCh <- readAtResponse{ + resCh <- readResponse{ Error: err, } return } size, err := io.ReadFull(httpReader, req.Buffer) if err == io.ErrUnexpectedEOF { - // If an EOF happens after reading some but not all the bytes - // ReadFull returns ErrUnexpectedEOF + // If an EOF happens after reading some but not + // all the bytes ReadFull returns ErrUnexpectedEOF err = io.EOF } - resCh <- readAtResponse{ + resCh <- readResponse{ Size: int(size), Error: err, } @@ -207,78 +199,148 @@ func (c Client) GetObjectPartial(bucketName, objectName string) (ReadAtCloser, O } }() // Return the readerAt backed by routine. - return newObjectReadAtCloser(reqCh, resCh, doneCh, objectStat.Size), objectStat, nil + return newObject(reqCh, resCh, doneCh, objectInfo), nil } -// response message container to reply back for the request. -type readAtResponse struct { +// Read response message container to reply back for the request. +type readResponse struct { Size int Error error } -// request message container to communicate with internal go-routine. -type readAtRequest struct { +// Read request message container to communicate with internal +// go-routine. +type readRequest struct { Buffer []byte Offset int64 // readAt offset. } -// objectReadAtCloser container for io.ReadAtCloser. -type objectReadAtCloser struct { - // mutex. +// Object represents an open object. It implements Read, ReadAt, +// Seeker, Close for a HTTP stream. +type Object struct { + // Mutex. mutex *sync.Mutex // User allocated and defined. - reqCh chan<- readAtRequest - resCh <-chan readAtResponse + reqCh chan<- readRequest + resCh <-chan readResponse doneCh chan<- struct{} - objectSize int64 + currOffset int64 + objectInfo ObjectInfo // Previous error saved for future calls. prevErr error } -// newObjectReadAtCloser implements a io.ReadSeeker for a HTTP stream. -func newObjectReadAtCloser(reqCh chan<- readAtRequest, resCh <-chan readAtResponse, doneCh chan<- struct{}, objectSize int64) *objectReadAtCloser { - return &objectReadAtCloser{ - mutex: new(sync.Mutex), - reqCh: reqCh, - resCh: resCh, - doneCh: doneCh, - objectSize: objectSize, +// Read reads up to len(p) bytes into p. It returns the number of +// bytes read (0 <= n <= len(p)) and any error encountered. Returns +// io.EOF upon end of file. +func (o *Object) Read(b []byte) (n int, err error) { + if o == nil { + return 0, ErrInvalidArgument("Object is nil") } + + // Locking. + o.mutex.Lock() + defer o.mutex.Unlock() + + // If current offset has reached Size limit, return EOF. + if o.currOffset >= o.objectInfo.Size { + return 0, io.EOF + } + + // Previous prevErr is which was saved in previous operation. + if o.prevErr != nil { + return 0, o.prevErr + } + + // Send current information over control channel to indicate we + // are ready. + reqMsg := readRequest{} + + // Send the offset and pointer to the buffer over the channel. + reqMsg.Buffer = b + reqMsg.Offset = o.currOffset + + // Send read request over the control channel. + o.reqCh <- reqMsg + + // Get data over the response channel. + dataMsg := <-o.resCh + + // Bytes read. + bytesRead := int64(dataMsg.Size) + + // Update current offset. + o.currOffset += bytesRead + + if dataMsg.Error == nil { + // If currOffset read is equal to objectSize + // We have reached end of file, we return io.EOF. + if o.currOffset >= o.objectInfo.Size { + return dataMsg.Size, io.EOF + } + return dataMsg.Size, nil + } + + // Save any error. + o.prevErr = dataMsg.Error + return dataMsg.Size, dataMsg.Error } -// ReadAt reads len(b) bytes from the File starting at byte offset off. -// It returns the number of bytes read and the error, if any. -// ReadAt always returns a non-nil error when n < len(b). -// At end of file, that error is io.EOF. -func (r *objectReadAtCloser) ReadAt(b []byte, offset int64) (int, error) { +// Stat returns the ObjectInfo structure describing object. +func (o *Object) Stat() (ObjectInfo, error) { + if o == nil { + return ObjectInfo{}, ErrInvalidArgument("Object is nil") + } // Locking. - r.mutex.Lock() - defer r.mutex.Unlock() + o.mutex.Lock() + defer o.mutex.Unlock() - // if offset is negative and offset is greater than or equal to object size we return EOF. - if offset < 0 || offset >= r.objectSize { + if o.prevErr != nil { + return ObjectInfo{}, o.prevErr + } + + return o.objectInfo, nil +} + +// ReadAt reads len(b) bytes from the File starting at byte offset +// off. It returns the number of bytes read and the error, if any. +// ReadAt always returns a non-nil error when n < len(b). At end of +// file, that error is io.EOF. +func (o *Object) ReadAt(b []byte, offset int64) (n int, err error) { + if o == nil { + return 0, ErrInvalidArgument("Object is nil") + } + + // Locking. + o.mutex.Lock() + defer o.mutex.Unlock() + + // If offset is negative and offset is greater than or equal to + // object size we return EOF. + if offset < 0 || offset >= o.objectInfo.Size { return 0, io.EOF } // prevErr is which was saved in previous operation. - if r.prevErr != nil { - return 0, r.prevErr + if o.prevErr != nil { + return 0, o.prevErr } - // Send current information over control channel to indicate we are ready. - reqMsg := readAtRequest{} + // Send current information over control channel to indicate we + // are ready. + reqMsg := readRequest{} - // Send the current offset and bytes requested. + // Send the offset and pointer to the buffer over the channel. reqMsg.Buffer = b reqMsg.Offset = offset // Send read request over the control channel. - r.reqCh <- reqMsg + o.reqCh <- reqMsg // Get data over the response channel. - dataMsg := <-r.resCh + dataMsg := <-o.resCh // Bytes read. bytesRead := int64(dataMsg.Size) @@ -286,38 +348,109 @@ func (r *objectReadAtCloser) ReadAt(b []byte, offset int64) (int, error) { if dataMsg.Error == nil { // If offset+bytes read is equal to objectSize // we have reached end of file, we return io.EOF. - if offset+bytesRead == r.objectSize { + if offset+bytesRead == o.objectInfo.Size { return dataMsg.Size, io.EOF } return dataMsg.Size, nil } // Save any error. - r.prevErr = dataMsg.Error + o.prevErr = dataMsg.Error return dataMsg.Size, dataMsg.Error } -// Closer is the interface that wraps the basic Close method. +// Seek sets the offset for the next Read or Write to offset, +// interpreted according to whence: 0 means relative to the +// origin of the file, 1 means relative to the current offset, +// and 2 means relative to the end. +// Seek returns the new offset and an error, if any. // -// The behavior of Close after the first call returns error for -// subsequent Close() calls. -func (r *objectReadAtCloser) Close() (err error) { +// Seeking to a negative offset is an error. Seeking to any positive +// offset is legal, subsequent io operations succeed until the +// underlying object is not closed. +func (o *Object) Seek(offset int64, whence int) (n int64, err error) { + if o == nil { + return 0, ErrInvalidArgument("Object is nil") + } + // Locking. - r.mutex.Lock() - defer r.mutex.Unlock() + o.mutex.Lock() + defer o.mutex.Unlock() + + if o.prevErr != nil { + // At EOF seeking is legal, for any other errors we return. + if o.prevErr != io.EOF { + return 0, o.prevErr + } + } + + // Negative offset is valid for whence of '2'. + if offset < 0 && whence != 2 { + return 0, ErrInvalidArgument(fmt.Sprintf("Object: negative position not allowed for %d.", whence)) + } + switch whence { + default: + return 0, ErrInvalidArgument(fmt.Sprintf("Object: invalid whence %d", whence)) + case 0: + if offset > o.objectInfo.Size { + return 0, io.EOF + } + o.currOffset = offset + case 1: + if o.currOffset+offset > o.objectInfo.Size { + return 0, io.EOF + } + o.currOffset += offset + case 2: + // Seeking to positive offset is valid for whence '2', but + // since we are backing a Reader we have reached 'EOF' if + // offset is positive. + if offset > 0 { + return 0, io.EOF + } + // Seeking to negative position not allowed for whence. + if o.objectInfo.Size+offset < 0 { + return 0, ErrInvalidArgument(fmt.Sprintf("Object: Seeking at negative offset not allowed for %d", whence)) + } + o.currOffset += offset + } + // Return the effective offset. + return o.currOffset, nil +} + +// Close - The behavior of Close after the first call returns error +// for subsequent Close() calls. +func (o *Object) Close() (err error) { + if o == nil { + return ErrInvalidArgument("Object is nil") + } + // Locking. + o.mutex.Lock() + defer o.mutex.Unlock() // prevErr is which was saved in previous operation. - if r.prevErr != nil { - return r.prevErr + if o.prevErr != nil { + return o.prevErr } // Close successfully. - close(r.doneCh) + close(o.doneCh) // Save this for any subsequent frivolous reads. - errMsg := "objectReadAtCloser: is already closed. Bad file descriptor." - r.prevErr = errors.New(errMsg) - return + errMsg := "Object: Is already closed. Bad file descriptor." + o.prevErr = errors.New(errMsg) + return nil +} + +// newObject instantiates a new *minio.Object* +func newObject(reqCh chan<- readRequest, resCh <-chan readResponse, doneCh chan<- struct{}, objectInfo ObjectInfo) *Object { + return &Object{ + mutex: &sync.Mutex{}, + reqCh: reqCh, + resCh: resCh, + doneCh: doneCh, + objectInfo: objectInfo, + } } // getObject - retrieve object from Object Storage. @@ -327,13 +460,13 @@ func (r *objectReadAtCloser) Close() (err error) { // // For more information about the HTTP Range header. // go to http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.35. -func (c Client) getObject(bucketName, objectName string, offset, length int64) (io.ReadCloser, ObjectStat, error) { +func (c Client) getObject(bucketName, objectName string, offset, length int64) (io.ReadCloser, ObjectInfo, error) { // Validate input arguments. if err := isValidBucketName(bucketName); err != nil { - return nil, ObjectStat{}, err + return nil, ObjectInfo{}, err } if err := isValidObjectName(objectName); err != nil { - return nil, ObjectStat{}, err + return nil, ObjectInfo{}, err } customHeader := make(http.Header) @@ -353,16 +486,16 @@ func (c Client) getObject(bucketName, objectName string, offset, length int64) ( customHeader: customHeader, }) if err != nil { - return nil, ObjectStat{}, err + return nil, ObjectInfo{}, err } // Execute the request. resp, err := c.do(req) if err != nil { - return nil, ObjectStat{}, err + return nil, ObjectInfo{}, err } if resp != nil { if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusPartialContent { - return nil, ObjectStat{}, HTTPRespToErrorResponse(resp, bucketName, objectName) + return nil, ObjectInfo{}, HTTPRespToErrorResponse(resp, bucketName, objectName) } } @@ -374,7 +507,7 @@ func (c Client) getObject(bucketName, objectName string, offset, length int64) ( date, err := time.Parse(http.TimeFormat, resp.Header.Get("Last-Modified")) if err != nil { msg := "Last-Modified time format not recognized. " + reportIssue - return nil, ObjectStat{}, ErrorResponse{ + return nil, ObjectInfo{}, ErrorResponse{ Code: "InternalError", Message: msg, RequestID: resp.Header.Get("x-amz-request-id"), @@ -387,7 +520,7 @@ func (c Client) getObject(bucketName, objectName string, offset, length int64) ( if contentType == "" { contentType = "application/octet-stream" } - var objectStat ObjectStat + var objectStat ObjectInfo objectStat.ETag = md5sum objectStat.Key = objectName objectStat.Size = resp.ContentLength diff --git a/Godeps/_workspace/src/github.com/minio/minio-go/api-list.go b/Godeps/_workspace/src/github.com/minio/minio-go/api-list.go index 8838900a8..7d0ffbaa6 100644 --- a/Godeps/_workspace/src/github.com/minio/minio-go/api-list.go +++ b/Godeps/_workspace/src/github.com/minio/minio-go/api-list.go @@ -33,7 +33,7 @@ import ( // fmt.Println(message) // } // -func (c Client) ListBuckets() ([]BucketStat, error) { +func (c Client) ListBuckets() ([]BucketInfo, error) { // Instantiate a new request. req, err := c.newRequest("GET", requestMetadata{}) if err != nil { @@ -64,19 +64,25 @@ func (c Client) ListBuckets() ([]BucketStat, error) { // the specified bucket. If recursion is enabled it would list // all subdirectories and all its contents. // -// Your input paramters are just bucketName, objectPrefix and recursive. If you -// enable recursive as 'true' this function will return back all the -// objects in a given bucket name and object prefix. +// Your input paramters are just bucketName, objectPrefix, recursive +// and a done channel for pro-actively closing the internal go +// routine. If you enable recursive as 'true' this function will +// return back all the objects in a given bucket name and object +// prefix. // // api := client.New(....) +// // Create a done channel. +// doneCh := make(chan struct{}) +// defer close(doneCh) +// // Recurively list all objects in 'mytestbucket' // recursive := true -// for message := range api.ListObjects("mytestbucket", "starthere", recursive) { +// for message := range api.ListObjects("mytestbucket", "starthere", recursive, doneCh) { // fmt.Println(message) // } // -func (c Client) ListObjects(bucketName, objectPrefix string, recursive bool, doneCh <-chan struct{}) <-chan ObjectStat { +func (c Client) ListObjects(bucketName, objectPrefix string, recursive bool, doneCh <-chan struct{}) <-chan ObjectInfo { // Allocate new list objects channel. - objectStatCh := make(chan ObjectStat, 1000) + objectStatCh := make(chan ObjectInfo, 1000) // Default listing is delimited at "/" delimiter := "/" if recursive { @@ -86,7 +92,7 @@ func (c Client) ListObjects(bucketName, objectPrefix string, recursive bool, don // Validate bucket name. if err := isValidBucketName(bucketName); err != nil { defer close(objectStatCh) - objectStatCh <- ObjectStat{ + objectStatCh <- ObjectInfo{ Err: err, } return objectStatCh @@ -94,14 +100,14 @@ func (c Client) ListObjects(bucketName, objectPrefix string, recursive bool, don // Validate incoming object prefix. if err := isValidObjectPrefix(objectPrefix); err != nil { defer close(objectStatCh) - objectStatCh <- ObjectStat{ + objectStatCh <- ObjectInfo{ Err: err, } return objectStatCh } // Initiate list objects goroutine here. - go func(objectStatCh chan<- ObjectStat) { + go func(objectStatCh chan<- ObjectInfo) { defer close(objectStatCh) // Save marker for next request. var marker string @@ -109,7 +115,7 @@ func (c Client) ListObjects(bucketName, objectPrefix string, recursive bool, don // Get list of objects a maximum of 1000 per request. result, err := c.listObjectsQuery(bucketName, objectPrefix, marker, delimiter, 1000) if err != nil { - objectStatCh <- ObjectStat{ + objectStatCh <- ObjectInfo{ Err: err, } return @@ -131,7 +137,7 @@ func (c Client) ListObjects(bucketName, objectPrefix string, recursive bool, don // Send all common prefixes if any. // NOTE: prefixes are only present if the request is delimited. for _, obj := range result.CommonPrefixes { - object := ObjectStat{} + object := ObjectInfo{} object.Key = obj.Prefix object.Size = 0 select { @@ -181,11 +187,22 @@ func (c Client) listObjectsQuery(bucketName, objectPrefix, objectMarker, delimit // using them in http request. urlValues := make(url.Values) // Set object prefix. - urlValues.Set("prefix", urlEncodePath(objectPrefix)) + if objectPrefix != "" { + urlValues.Set("prefix", urlEncodePath(objectPrefix)) + } // Set object marker. - urlValues.Set("marker", urlEncodePath(objectMarker)) + if objectMarker != "" { + urlValues.Set("marker", urlEncodePath(objectMarker)) + } // Set delimiter. - urlValues.Set("delimiter", delimiter) + if delimiter != "" { + urlValues.Set("delimiter", delimiter) + } + + // maxkeys should default to 1000 or less. + if maxkeys == 0 || maxkeys > 1000 { + maxkeys = 1000 + } // Set max keys. urlValues.Set("max-keys", fmt.Sprintf("%d", maxkeys)) @@ -223,26 +240,31 @@ func (c Client) listObjectsQuery(bucketName, objectPrefix, objectMarker, delimit // objectPrefix from the specified bucket. If recursion is enabled // it would list all subdirectories and all its contents. // -// Your input paramters are just bucketName, objectPrefix and recursive. +// Your input paramters are just bucketName, objectPrefix, recursive +// and a done channel to proactively close the internal go routine. // If you enable recursive as 'true' this function will return back all // the multipart objects in a given bucket name. // // api := client.New(....) +// // Create a done channel. +// doneCh := make(chan struct{}) +// defer close(doneCh) +// // Recurively list all objects in 'mytestbucket' // recursive := true // for message := range api.ListIncompleteUploads("mytestbucket", "starthere", recursive) { // fmt.Println(message) // } // -func (c Client) ListIncompleteUploads(bucketName, objectPrefix string, recursive bool, doneCh <-chan struct{}) <-chan ObjectMultipartStat { +func (c Client) ListIncompleteUploads(bucketName, objectPrefix string, recursive bool, doneCh <-chan struct{}) <-chan ObjectMultipartInfo { // Turn on size aggregation of individual parts. isAggregateSize := true return c.listIncompleteUploads(bucketName, objectPrefix, recursive, isAggregateSize, doneCh) } // listIncompleteUploads lists all incomplete uploads. -func (c Client) listIncompleteUploads(bucketName, objectPrefix string, recursive, aggregateSize bool, doneCh <-chan struct{}) <-chan ObjectMultipartStat { +func (c Client) listIncompleteUploads(bucketName, objectPrefix string, recursive, aggregateSize bool, doneCh <-chan struct{}) <-chan ObjectMultipartInfo { // Allocate channel for multipart uploads. - objectMultipartStatCh := make(chan ObjectMultipartStat, 1000) + objectMultipartStatCh := make(chan ObjectMultipartInfo, 1000) // Delimiter is set to "/" by default. delimiter := "/" if recursive { @@ -252,7 +274,7 @@ func (c Client) listIncompleteUploads(bucketName, objectPrefix string, recursive // Validate bucket name. if err := isValidBucketName(bucketName); err != nil { defer close(objectMultipartStatCh) - objectMultipartStatCh <- ObjectMultipartStat{ + objectMultipartStatCh <- ObjectMultipartInfo{ Err: err, } return objectMultipartStatCh @@ -260,12 +282,12 @@ func (c Client) listIncompleteUploads(bucketName, objectPrefix string, recursive // Validate incoming object prefix. if err := isValidObjectPrefix(objectPrefix); err != nil { defer close(objectMultipartStatCh) - objectMultipartStatCh <- ObjectMultipartStat{ + objectMultipartStatCh <- ObjectMultipartInfo{ Err: err, } return objectMultipartStatCh } - go func(objectMultipartStatCh chan<- ObjectMultipartStat) { + go func(objectMultipartStatCh chan<- ObjectMultipartInfo) { defer close(objectMultipartStatCh) // object and upload ID marker for future requests. var objectMarker string @@ -274,7 +296,7 @@ func (c Client) listIncompleteUploads(bucketName, objectPrefix string, recursive // list all multipart uploads. result, err := c.listMultipartUploadsQuery(bucketName, objectMarker, uploadIDMarker, objectPrefix, delimiter, 1000) if err != nil { - objectMultipartStatCh <- ObjectMultipartStat{ + objectMultipartStatCh <- ObjectMultipartInfo{ Err: err, } return @@ -289,7 +311,7 @@ func (c Client) listIncompleteUploads(bucketName, objectPrefix string, recursive // Get total multipart size. obj.Size, err = c.getTotalMultipartSize(bucketName, obj.Key, obj.UploadID) if err != nil { - objectMultipartStatCh <- ObjectMultipartStat{ + objectMultipartStatCh <- ObjectMultipartInfo{ Err: err, } } @@ -305,7 +327,7 @@ func (c Client) listIncompleteUploads(bucketName, objectPrefix string, recursive // Send all common prefixes if any. // NOTE: prefixes are only present if the request is delimited. for _, obj := range result.CommonPrefixes { - object := ObjectMultipartStat{} + object := ObjectMultipartInfo{} object.Key = obj.Prefix object.Size = 0 select { @@ -343,13 +365,26 @@ func (c Client) listMultipartUploadsQuery(bucketName, keyMarker, uploadIDMarker, // Set uploads. urlValues.Set("uploads", "") // Set object key marker. - urlValues.Set("key-marker", urlEncodePath(keyMarker)) + if keyMarker != "" { + urlValues.Set("key-marker", urlEncodePath(keyMarker)) + } // Set upload id marker. - urlValues.Set("upload-id-marker", uploadIDMarker) + if uploadIDMarker != "" { + urlValues.Set("upload-id-marker", uploadIDMarker) + } // Set prefix marker. - urlValues.Set("prefix", urlEncodePath(prefix)) + if prefix != "" { + urlValues.Set("prefix", urlEncodePath(prefix)) + } // Set delimiter. - urlValues.Set("delimiter", delimiter) + if delimiter != "" { + urlValues.Set("delimiter", delimiter) + } + + // maxUploads should be 1000 or less. + if maxUploads == 0 || maxUploads > 1000 { + maxUploads = 1000 + } // Set max-uploads. urlValues.Set("max-uploads", fmt.Sprintf("%d", maxUploads)) @@ -445,12 +480,15 @@ func (c Client) getTotalMultipartSize(bucketName, objectName, uploadID string) ( } // listObjectPartsQuery (List Parts query) -// - lists some or all (up to 1000) parts that have been uploaded for a specific multipart upload +// - lists some or all (up to 1000) parts that have been uploaded +// for a specific multipart upload // -// You can use the request parameters as selection criteria to return a subset of the uploads in a bucket. -// request paramters :- +// You can use the request parameters as selection criteria to return +// a subset of the uploads in a bucket, request paramters :- // --------- -// ?part-number-marker - Specifies the part after which listing should begin. +// ?part-number-marker - Specifies the part after which listing should +// begin. +// ?max-parts - Maximum parts to be listed per request. func (c Client) listObjectPartsQuery(bucketName, objectName, uploadID string, partNumberMarker, maxParts int) (listObjectPartsResult, error) { // Get resources properly escaped and lined up before using them in http request. urlValues := make(url.Values) @@ -458,6 +496,11 @@ func (c Client) listObjectPartsQuery(bucketName, objectName, uploadID string, pa urlValues.Set("part-number-marker", fmt.Sprintf("%d", partNumberMarker)) // Set upload id. urlValues.Set("uploadId", uploadID) + + // maxParts should be 1000 or less. + if maxParts == 0 || maxParts > 1000 { + maxParts = 1000 + } // Set max parts. urlValues.Set("max-parts", fmt.Sprintf("%d", maxParts)) diff --git a/Godeps/_workspace/src/github.com/minio/minio-go/api-presigned.go b/Godeps/_workspace/src/github.com/minio/minio-go/api-presigned.go index d46623631..e1b40e0e3 100644 --- a/Godeps/_workspace/src/github.com/minio/minio-go/api-presigned.go +++ b/Godeps/_workspace/src/github.com/minio/minio-go/api-presigned.go @@ -21,7 +21,7 @@ import ( "time" ) -// PresignedGetObject returns a presigned URL to access an object without credentials. +// PresignedGetObject - Returns a presigned URL to access an object without credentials. // Expires maximum is 7days - ie. 604800 and minimum is 1. func (c Client) PresignedGetObject(bucketName, objectName string, expires time.Duration) (string, error) { // Input validation. @@ -50,7 +50,7 @@ func (c Client) PresignedGetObject(bucketName, objectName string, expires time.D return req.URL.String(), nil } -// PresignedPutObject returns a presigned URL to upload an object without credentials. +// PresignedPutObject - Returns a presigned URL to upload an object without credentials. // Expires maximum is 7days - ie. 604800 and minimum is 1. func (c Client) PresignedPutObject(bucketName, objectName string, expires time.Duration) (string, error) { // Input validation. @@ -79,7 +79,7 @@ func (c Client) PresignedPutObject(bucketName, objectName string, expires time.D return req.URL.String(), nil } -// PresignedPostPolicy returns POST form data to upload an object at a location. +// PresignedPostPolicy - Returns POST form data to upload an object at a location. func (c Client) PresignedPostPolicy(p *PostPolicy) (map[string]string, error) { // Validate input arguments. if p.expiration.IsZero() { @@ -93,7 +93,7 @@ func (c Client) PresignedPostPolicy(p *PostPolicy) (map[string]string, error) { } bucketName := p.formData["bucket"] - // Fetch the location. + // Fetch the bucket location. location, err := c.getBucketLocation(bucketName) if err != nil { return nil, err @@ -101,6 +101,7 @@ func (c Client) PresignedPostPolicy(p *PostPolicy) (map[string]string, error) { // Keep time. t := time.Now().UTC() + // For signature version '2' handle here. if c.signature.isV2() { policyBase64 := p.base64() p.formData["policy"] = policyBase64 @@ -135,7 +136,7 @@ func (c Client) PresignedPostPolicy(p *PostPolicy) (map[string]string, error) { condition: "$x-amz-credential", value: credential, }) - // get base64 encoded policy. + // Get base64 encoded policy. policyBase64 := p.base64() // Fill in the form data. p.formData["policy"] = policyBase64 diff --git a/Godeps/_workspace/src/github.com/minio/minio-go/api-put-bucket.go b/Godeps/_workspace/src/github.com/minio/minio-go/api-put-bucket.go index e1880d9f8..07648a24d 100644 --- a/Godeps/_workspace/src/github.com/minio/minio-go/api-put-bucket.go +++ b/Godeps/_workspace/src/github.com/minio/minio-go/api-put-bucket.go @@ -97,7 +97,7 @@ func (c Client) makeBucketRequest(bucketName string, acl BucketACL, location str } // Set get bucket location always as path style. - targetURL := c.endpointURL + targetURL := *c.endpointURL if bucketName != "" { // If endpoint supports virtual host style use that always. // Currently only S3 and Google Cloud Storage would support this. @@ -132,7 +132,7 @@ func (c Client) makeBucketRequest(bucketName string, acl BucketACL, location str // If location is not 'us-east-1' create bucket location config. if location != "us-east-1" && location != "" { - createBucketConfig := new(createBucketConfiguration) + createBucketConfig := createBucketConfiguration{} createBucketConfig.Location = location var createBucketConfigBytes []byte createBucketConfigBytes, err = xml.Marshal(createBucketConfig) diff --git a/Godeps/_workspace/src/github.com/minio/minio-go/api-fput-object.go b/Godeps/_workspace/src/github.com/minio/minio-go/api-put-object-file.go similarity index 67% rename from Godeps/_workspace/src/github.com/minio/minio-go/api-fput-object.go rename to Godeps/_workspace/src/github.com/minio/minio-go/api-put-object-file.go index 00b10aabb..5bc92d3bc 100644 --- a/Godeps/_workspace/src/github.com/minio/minio-go/api-fput-object.go +++ b/Godeps/_workspace/src/github.com/minio/minio-go/api-put-object-file.go @@ -28,7 +28,8 @@ import ( "sort" ) -// getUploadID if already present for object name or initiate a request to fetch a new upload id. +// getUploadID - fetch upload id if already present for an object name +// or initiate a new request to fetch a new upload id. func (c Client) getUploadID(bucketName, objectName, contentType string) (string, error) { // Input validation. if err := isValidBucketName(bucketName); err != nil { @@ -60,84 +61,16 @@ func (c Client) getUploadID(bucketName, objectName, contentType string) (string, return uploadID, nil } -// FPutObject - put object a file. -func (c Client) FPutObject(bucketName, objectName, filePath, contentType string) (int64, error) { - // Input validation. - if err := isValidBucketName(bucketName); err != nil { - return 0, err - } - if err := isValidObjectName(objectName); err != nil { - return 0, err - } - - // Open the referenced file. - fileData, err := os.Open(filePath) - // If any error fail quickly here. - if err != nil { - return 0, err - } - defer fileData.Close() - - // Save the file stat. - fileStat, err := fileData.Stat() - if err != nil { - return 0, err - } - - // Save the file size. - fileSize := fileStat.Size() - if fileSize > int64(maxMultipartPutObjectSize) { - return 0, ErrInvalidArgument("Input file size is bigger than the supported maximum of 5TiB.") - } - - // NOTE: Google Cloud Storage multipart Put is not compatible with Amazon S3 APIs. - // Current implementation will only upload a maximum of 5GiB to Google Cloud Storage servers. - if isGoogleEndpoint(c.endpointURL) { - if fileSize > int64(maxSinglePutObjectSize) { - return 0, ErrorResponse{ - Code: "NotImplemented", - Message: fmt.Sprintf("Invalid Content-Length %d for file uploads to Google Cloud Storage.", fileSize), - Key: objectName, - BucketName: bucketName, - } - } - // Do not compute MD5 for Google Cloud Storage. Uploads upto 5GiB in size. - n, err := c.putNoChecksum(bucketName, objectName, fileData, fileSize, contentType) - return n, err - } - - // NOTE: S3 doesn't allow anonymous multipart requests. - if isAmazonEndpoint(c.endpointURL) && c.anonymous { - if fileSize > int64(maxSinglePutObjectSize) { - return 0, ErrorResponse{ - Code: "NotImplemented", - Message: fmt.Sprintf("For anonymous requests Content-Length cannot be %d.", fileSize), - Key: objectName, - BucketName: bucketName, - } - } - // Do not compute MD5 for anonymous requests to Amazon S3. Uploads upto 5GiB in size. - n, err := c.putAnonymous(bucketName, objectName, fileData, fileSize, contentType) - return n, err - } - - // Small object upload is initiated for uploads for input data size smaller than 5MiB. - if fileSize < minimumPartSize { - return c.putSmallObject(bucketName, objectName, fileData, fileSize, contentType) - } - return c.fputLargeObject(bucketName, objectName, fileData, fileSize, contentType) -} - -// computeHash - calculates MD5 and Sha256 for an input read Seeker. +// computeHash - Calculates MD5 and SHA256 for an input read Seeker. func (c Client) computeHash(reader io.ReadSeeker) (md5Sum, sha256Sum []byte, size int64, err error) { - // MD5 and Sha256 hasher. - var hashMD5, hashSha256 hash.Hash - // MD5 and Sha256 hasher. + // MD5 and SHA256 hasher. + var hashMD5, hashSHA256 hash.Hash + // MD5 and SHA256 hasher. hashMD5 = md5.New() hashWriter := io.MultiWriter(hashMD5) if c.signature.isV4() { - hashSha256 = sha256.New() - hashWriter = io.MultiWriter(hashMD5, hashSha256) + hashSHA256 = sha256.New() + hashWriter = io.MultiWriter(hashMD5, hashSHA256) } size, err = io.Copy(hashWriter, reader) @@ -153,12 +86,13 @@ func (c Client) computeHash(reader io.ReadSeeker) (md5Sum, sha256Sum []byte, siz // Finalize md5shum and sha256 sum. md5Sum = hashMD5.Sum(nil) if c.signature.isV4() { - sha256Sum = hashSha256.Sum(nil) + sha256Sum = hashSHA256.Sum(nil) } return md5Sum, sha256Sum, size, nil } -func (c Client) fputLargeObject(bucketName, objectName string, fileData *os.File, fileSize int64, contentType string) (int64, error) { +// FPutObject - Create an object in a bucket, with contents from file at filePath. +func (c Client) FPutObject(bucketName, objectName, filePath, contentType string) (n int64, err error) { // Input validation. if err := isValidBucketName(bucketName); err != nil { return 0, err @@ -167,27 +101,119 @@ func (c Client) fputLargeObject(bucketName, objectName string, fileData *os.File return 0, err } - // getUploadID for an object, initiates a new multipart request + // Open the referenced file. + fileReader, err := os.Open(filePath) + // If any error fail quickly here. + if err != nil { + return 0, err + } + defer fileReader.Close() + + // Save the file stat. + fileStat, err := fileReader.Stat() + if err != nil { + return 0, err + } + + // Save the file size. + fileSize := fileStat.Size() + + // Check for largest object size allowed. + if fileSize > int64(maxMultipartPutObjectSize) { + return 0, ErrEntityTooLarge(fileSize, bucketName, objectName) + } + + // NOTE: Google Cloud Storage multipart Put is not compatible with Amazon S3 APIs. + // Current implementation will only upload a maximum of 5GiB to Google Cloud Storage servers. + if isGoogleEndpoint(c.endpointURL) { + if fileSize > int64(maxSinglePutObjectSize) { + return 0, ErrorResponse{ + Code: "NotImplemented", + Message: fmt.Sprintf("Invalid Content-Length %d for file uploads to Google Cloud Storage.", fileSize), + Key: objectName, + BucketName: bucketName, + } + } + // Do not compute MD5 for Google Cloud Storage. Uploads upto 5GiB in size. + return c.putObjectNoChecksum(bucketName, objectName, fileReader, fileSize, contentType) + } + + // NOTE: S3 doesn't allow anonymous multipart requests. + if isAmazonEndpoint(c.endpointURL) && c.anonymous { + if fileSize > int64(maxSinglePutObjectSize) { + return 0, ErrorResponse{ + Code: "NotImplemented", + Message: fmt.Sprintf("For anonymous requests Content-Length cannot be %d.", fileSize), + Key: objectName, + BucketName: bucketName, + } + } + // Do not compute MD5 for anonymous requests to Amazon S3. Uploads upto 5GiB in size. + return c.putObjectNoChecksum(bucketName, objectName, fileReader, fileSize, contentType) + } + + // Small object upload is initiated for uploads for input data size smaller than 5MiB. + if fileSize < minimumPartSize { + return c.putObjectSingle(bucketName, objectName, fileReader, fileSize, contentType) + } + // Upload all large objects as multipart. + n, err = c.putObjectMultipartFromFile(bucketName, objectName, fileReader, fileSize, contentType) + if err != nil { + errResp := ToErrorResponse(err) + // Verify if multipart functionality is not available, if not + // fall back to single PutObject operation. + if errResp.Code == "NotImplemented" { + // If size of file is greater than '5GiB' fail. + if fileSize > maxSinglePutObjectSize { + return 0, ErrEntityTooLarge(fileSize, bucketName, objectName) + } + // Fall back to uploading as single PutObject operation. + return c.putObjectSingle(bucketName, objectName, fileReader, fileSize, contentType) + } + return n, err + } + return n, nil +} + +// putObjectMultipartFromFile - Creates object from contents of *os.File +// +// NOTE: This function is meant to be used for readers with local +// file as in *os.File. This function resumes by skipping all the +// necessary parts which were already uploaded by verifying them +// against MD5SUM of each individual parts. This function also +// effectively utilizes file system capabilities of reading from +// specific sections and not having to create temporary files. +func (c Client) putObjectMultipartFromFile(bucketName, objectName string, fileReader *os.File, fileSize int64, contentType string) (int64, error) { + // Input validation. + if err := isValidBucketName(bucketName); err != nil { + return 0, err + } + if err := isValidObjectName(objectName); err != nil { + return 0, err + } + + // Get upload id for an object, initiates a new multipart request // if it cannot find any previously partially uploaded object. uploadID, err := c.getUploadID(bucketName, objectName, contentType) if err != nil { return 0, err } - // total data read and written to server. should be equal to 'size' at the end of the call. + // Total data read and written to server. should be equal to 'size' at the end of the call. var totalUploadedSize int64 // Complete multipart upload. var completeMultipartUpload completeMultipartUpload - // Fetch previously upload parts and save the total size. + // Fetch previously upload parts. partsInfo, err := c.listObjectParts(bucketName, objectName, uploadID) if err != nil { return 0, err } + // Previous maximum part size var prevMaxPartSize int64 - // Loop through all parts and calculate totalUploadedSize. + // Loop through all parts and fetch prevMaxPartSize. for _, partInfo := range partsInfo { // Choose the maximum part size. if partInfo.Size >= prevMaxPartSize { @@ -197,7 +223,7 @@ func (c Client) fputLargeObject(bucketName, objectName string, fileData *os.File // Calculate the optimal part size for a given file size. partSize := optimalPartSize(fileSize) - // If prevMaxPartSize is set use that. + // Use prevMaxPartSize if available. if prevMaxPartSize != 0 { partSize = prevMaxPartSize } @@ -205,52 +231,39 @@ func (c Client) fputLargeObject(bucketName, objectName string, fileData *os.File // Part number always starts with '0'. partNumber := 0 - // Loop through until EOF. + // Upload each part until fileSize. for totalUploadedSize < fileSize { // Increment part number. partNumber++ // Get a section reader on a particular offset. - sectionReader := io.NewSectionReader(fileData, totalUploadedSize, partSize) + sectionReader := io.NewSectionReader(fileReader, totalUploadedSize, partSize) - // Calculates MD5 and Sha256 sum for a section reader. + // Calculates MD5 and SHA256 sum for a section reader. md5Sum, sha256Sum, size, err := c.computeHash(sectionReader) if err != nil { return 0, err } - // Save all the part metadata. - prtData := partData{ - ReadCloser: ioutil.NopCloser(sectionReader), - Size: size, - MD5Sum: md5Sum, - Sha256Sum: sha256Sum, - Number: partNumber, // Part number to be uploaded. - } - - // If part not uploaded proceed to upload. + // Verify if part was not uploaded. if !isPartUploaded(objectPart{ - ETag: hex.EncodeToString(prtData.MD5Sum), - PartNumber: prtData.Number, + ETag: hex.EncodeToString(md5Sum), + PartNumber: partNumber, }, partsInfo) { - // Upload the part. - objPart, err := c.uploadPart(bucketName, objectName, uploadID, prtData) + // Proceed to upload the part. + objPart, err := c.uploadPart(bucketName, objectName, uploadID, ioutil.NopCloser(sectionReader), partNumber, md5Sum, sha256Sum, size) if err != nil { - prtData.ReadCloser.Close() return totalUploadedSize, err } // Save successfully uploaded part metadata. - partsInfo[prtData.Number] = objPart + partsInfo[partNumber] = objPart } - // Close the read closer for temporary file. - prtData.ReadCloser.Close() - // Save successfully uploaded size. - totalUploadedSize += prtData.Size + totalUploadedSize += size } - // if totalUploadedSize is different than the file 'size'. Do not complete the request throw an error. + // Verify if we uploaded all data. if totalUploadedSize != fileSize { return totalUploadedSize, ErrUnexpectedEOF(totalUploadedSize, fileSize, bucketName, objectName) } @@ -263,7 +276,7 @@ func (c Client) fputLargeObject(bucketName, objectName string, fileData *os.File completeMultipartUpload.Parts = append(completeMultipartUpload.Parts, complPart) } - // If partNumber is different than total list of parts, error out. + // Verify if partNumber is different than total list of parts. if partNumber != len(completeMultipartUpload.Parts) { return totalUploadedSize, ErrInvalidParts(partNumber, len(completeMultipartUpload.Parts)) } diff --git a/Godeps/_workspace/src/github.com/minio/minio-go/api-put-object-multipart.go b/Godeps/_workspace/src/github.com/minio/minio-go/api-put-object-multipart.go new file mode 100644 index 000000000..6cacc9800 --- /dev/null +++ b/Godeps/_workspace/src/github.com/minio/minio-go/api-put-object-multipart.go @@ -0,0 +1,421 @@ +/* + * Minio Go Library for Amazon S3 Compatible Cloud Storage (C) 2015 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package minio + +import ( + "bytes" + "crypto/md5" + "crypto/sha256" + "encoding/hex" + "encoding/xml" + "hash" + "io" + "io/ioutil" + "net/http" + "net/url" + "os" + "sort" + "strconv" + "strings" +) + +// Verify if reader is *os.File +func isFile(reader io.Reader) (ok bool) { + _, ok = reader.(*os.File) + return +} + +// Verify if reader is *minio.Object +func isObject(reader io.Reader) (ok bool) { + _, ok = reader.(*Object) + return +} + +// Verify if reader is a generic ReaderAt +func isReadAt(reader io.Reader) (ok bool) { + _, ok = reader.(io.ReaderAt) + return +} + +// hashCopyN - Calculates Md5sum and SHA256sum for upto partSize amount of bytes. +func (c Client) hashCopyN(writer io.ReadWriteSeeker, reader io.Reader, partSize int64) (md5Sum, sha256Sum []byte, size int64, err error) { + // MD5 and SHA256 hasher. + var hashMD5, hashSHA256 hash.Hash + // MD5 and SHA256 hasher. + hashMD5 = md5.New() + hashWriter := io.MultiWriter(writer, hashMD5) + if c.signature.isV4() { + hashSHA256 = sha256.New() + hashWriter = io.MultiWriter(writer, hashMD5, hashSHA256) + } + + // Copies to input at writer. + size, err = io.CopyN(hashWriter, reader, partSize) + if err != nil { + // If not EOF return error right here. + if err != io.EOF { + return nil, nil, 0, err + } + } + + // Seek back to beginning of input, any error fail right here. + if _, err := writer.Seek(0, 0); err != nil { + return nil, nil, 0, err + } + + // Finalize md5shum and sha256 sum. + md5Sum = hashMD5.Sum(nil) + if c.signature.isV4() { + sha256Sum = hashSHA256.Sum(nil) + } + return md5Sum, sha256Sum, size, err +} + +// Comprehensive put object operation involving multipart resumable uploads. +// +// Following code handles these types of readers. +// +// - *os.File +// - *minio.Object +// - Any reader which has a method 'ReadAt()' +// +// If we exhaust all the known types, code proceeds to use stream as +// is where each part is re-downloaded, checksummed and verified +// before upload. +func (c Client) putObjectMultipart(bucketName, objectName string, reader io.Reader, size int64, contentType string) (n int64, err error) { + if size > 0 && size >= minimumPartSize { + // Verify if reader is *os.File, then use file system functionalities. + if isFile(reader) { + return c.putObjectMultipartFromFile(bucketName, objectName, reader.(*os.File), size, contentType) + } + // Verify if reader is *minio.Object or io.ReaderAt. + // NOTE: Verification of object is kept for a specific purpose + // while it is going to be duck typed similar to io.ReaderAt. + // It is to indicate that *minio.Object implements io.ReaderAt. + // and such a functionality is used in the subsequent code + // path. + if isObject(reader) || isReadAt(reader) { + return c.putObjectMultipartFromReadAt(bucketName, objectName, reader.(io.ReaderAt), size, contentType) + } + } + // For any other data size and reader type we do generic multipart + // approach by staging data in temporary files and uploading them. + return c.putObjectMultipartStream(bucketName, objectName, reader, size, contentType) +} + +// putObjectStream uploads files bigger than 5MiB, and also supports +// special case where size is unknown i.e '-1'. +func (c Client) putObjectMultipartStream(bucketName, objectName string, reader io.Reader, size int64, contentType string) (n int64, err error) { + // Input validation. + if err := isValidBucketName(bucketName); err != nil { + return 0, err + } + if err := isValidObjectName(objectName); err != nil { + return 0, err + } + + // getUploadID for an object, initiates a new multipart request + // if it cannot find any previously partially uploaded object. + uploadID, err := c.getUploadID(bucketName, objectName, contentType) + if err != nil { + return 0, err + } + + // Total data read and written to server. should be equal to 'size' at the end of the call. + var totalUploadedSize int64 + + // Complete multipart upload. + var completeMultipartUpload completeMultipartUpload + + // Fetch previously upload parts. + partsInfo, err := c.listObjectParts(bucketName, objectName, uploadID) + if err != nil { + return 0, err + } + // Previous maximum part size + var prevMaxPartSize int64 + // Loop through all parts and calculate totalUploadedSize. + for _, partInfo := range partsInfo { + // Choose the maximum part size. + if partInfo.Size >= prevMaxPartSize { + prevMaxPartSize = partInfo.Size + } + } + + // Calculate the optimal part size for a given size. + partSize := optimalPartSize(size) + // Use prevMaxPartSize if available. + if prevMaxPartSize != 0 { + partSize = prevMaxPartSize + } + + // Part number always starts with '0'. + partNumber := 0 + + // Upload each part until EOF. + for { + // Increment part number. + partNumber++ + + // Initialize a new temporary file. + tmpFile, err := newTempFile("multiparts$-putobject-stream") + if err != nil { + return 0, err + } + + // Calculates MD5 and SHA256 sum while copying partSize bytes into tmpFile. + md5Sum, sha256Sum, size, rErr := c.hashCopyN(tmpFile, reader, partSize) + if rErr != nil { + if rErr != io.EOF { + return 0, rErr + } + } + + // Verify if part was not uploaded. + if !isPartUploaded(objectPart{ + ETag: hex.EncodeToString(md5Sum), + PartNumber: partNumber, + }, partsInfo) { + // Proceed to upload the part. + objPart, err := c.uploadPart(bucketName, objectName, uploadID, tmpFile, partNumber, md5Sum, sha256Sum, size) + if err != nil { + // Close the temporary file upon any error. + tmpFile.Close() + return 0, err + } + // Save successfully uploaded part metadata. + partsInfo[partNumber] = objPart + } + + // Close the temporary file. + tmpFile.Close() + + // If read error was an EOF, break out of the loop. + if rErr == io.EOF { + break + } + } + + // Verify if we uploaded all the data. + if size > 0 { + if totalUploadedSize != size { + return totalUploadedSize, ErrUnexpectedEOF(totalUploadedSize, size, bucketName, objectName) + } + } + + // Loop over uploaded parts to save them in a Parts array before completing the multipart request. + for _, part := range partsInfo { + var complPart completePart + complPart.ETag = part.ETag + complPart.PartNumber = part.PartNumber + completeMultipartUpload.Parts = append(completeMultipartUpload.Parts, complPart) + // Save successfully uploaded size. + totalUploadedSize += part.Size + } + + // Verify if partNumber is different than total list of parts. + if partNumber != len(completeMultipartUpload.Parts) { + return totalUploadedSize, ErrInvalidParts(partNumber, len(completeMultipartUpload.Parts)) + } + + // Sort all completed parts. + sort.Sort(completedParts(completeMultipartUpload.Parts)) + _, err = c.completeMultipartUpload(bucketName, objectName, uploadID, completeMultipartUpload) + if err != nil { + return totalUploadedSize, err + } + + // Return final size. + return totalUploadedSize, nil +} + +// initiateMultipartUpload - Initiates a multipart upload and returns an upload ID. +func (c Client) initiateMultipartUpload(bucketName, objectName, contentType string) (initiateMultipartUploadResult, error) { + // Input validation. + if err := isValidBucketName(bucketName); err != nil { + return initiateMultipartUploadResult{}, err + } + if err := isValidObjectName(objectName); err != nil { + return initiateMultipartUploadResult{}, err + } + + // Initialize url queries. + urlValues := make(url.Values) + urlValues.Set("uploads", "") + + if contentType == "" { + contentType = "application/octet-stream" + } + + // Set ContentType header. + customHeader := make(http.Header) + customHeader.Set("Content-Type", contentType) + + reqMetadata := requestMetadata{ + bucketName: bucketName, + objectName: objectName, + queryValues: urlValues, + customHeader: customHeader, + } + + // Instantiate the request. + req, err := c.newRequest("POST", reqMetadata) + if err != nil { + return initiateMultipartUploadResult{}, err + } + + // Execute the request. + resp, err := c.do(req) + defer closeResponse(resp) + if err != nil { + return initiateMultipartUploadResult{}, err + } + if resp != nil { + if resp.StatusCode != http.StatusOK { + return initiateMultipartUploadResult{}, HTTPRespToErrorResponse(resp, bucketName, objectName) + } + } + // Decode xml for new multipart upload. + initiateMultipartUploadResult := initiateMultipartUploadResult{} + err = xmlDecoder(resp.Body, &initiateMultipartUploadResult) + if err != nil { + return initiateMultipartUploadResult, err + } + return initiateMultipartUploadResult, nil +} + +// uploadPart - Uploads a part in a multipart upload. +func (c Client) uploadPart(bucketName, objectName, uploadID string, reader io.ReadCloser, partNumber int, md5Sum, sha256Sum []byte, size int64) (objectPart, error) { + // Input validation. + if err := isValidBucketName(bucketName); err != nil { + return objectPart{}, err + } + if err := isValidObjectName(objectName); err != nil { + return objectPart{}, err + } + if size > maxPartSize { + return objectPart{}, ErrEntityTooLarge(size, bucketName, objectName) + } + if size <= -1 { + return objectPart{}, ErrEntityTooSmall(size, bucketName, objectName) + } + if partNumber <= 0 { + return objectPart{}, ErrInvalidArgument("Part number cannot be negative or equal to zero.") + } + if uploadID == "" { + return objectPart{}, ErrInvalidArgument("UploadID cannot be empty.") + } + + // Get resources properly escaped and lined up before using them in http request. + urlValues := make(url.Values) + // Set part number. + urlValues.Set("partNumber", strconv.Itoa(partNumber)) + // Set upload id. + urlValues.Set("uploadId", uploadID) + + reqMetadata := requestMetadata{ + bucketName: bucketName, + objectName: objectName, + queryValues: urlValues, + contentBody: reader, + contentLength: size, + contentMD5Bytes: md5Sum, + contentSHA256Bytes: sha256Sum, + } + + // Instantiate a request. + req, err := c.newRequest("PUT", reqMetadata) + if err != nil { + return objectPart{}, err + } + // Execute the request. + resp, err := c.do(req) + defer closeResponse(resp) + if err != nil { + return objectPart{}, err + } + if resp != nil { + if resp.StatusCode != http.StatusOK { + return objectPart{}, HTTPRespToErrorResponse(resp, bucketName, objectName) + } + } + // Once successfully uploaded, return completed part. + objPart := objectPart{} + objPart.Size = size + objPart.PartNumber = partNumber + // Trim off the odd double quotes from ETag in the beginning and end. + objPart.ETag = strings.TrimPrefix(resp.Header.Get("ETag"), "\"") + objPart.ETag = strings.TrimSuffix(objPart.ETag, "\"") + return objPart, nil +} + +// completeMultipartUpload - Completes a multipart upload by assembling previously uploaded parts. +func (c Client) completeMultipartUpload(bucketName, objectName, uploadID string, complete completeMultipartUpload) (completeMultipartUploadResult, error) { + // Input validation. + if err := isValidBucketName(bucketName); err != nil { + return completeMultipartUploadResult{}, err + } + if err := isValidObjectName(objectName); err != nil { + return completeMultipartUploadResult{}, err + } + + // Initialize url queries. + urlValues := make(url.Values) + urlValues.Set("uploadId", uploadID) + + // Marshal complete multipart body. + completeMultipartUploadBytes, err := xml.Marshal(complete) + if err != nil { + return completeMultipartUploadResult{}, err + } + + // Instantiate all the complete multipart buffer. + completeMultipartUploadBuffer := bytes.NewBuffer(completeMultipartUploadBytes) + reqMetadata := requestMetadata{ + bucketName: bucketName, + objectName: objectName, + queryValues: urlValues, + contentBody: ioutil.NopCloser(completeMultipartUploadBuffer), + contentLength: int64(completeMultipartUploadBuffer.Len()), + contentSHA256Bytes: sum256(completeMultipartUploadBuffer.Bytes()), + } + + // Instantiate the request. + req, err := c.newRequest("POST", reqMetadata) + if err != nil { + return completeMultipartUploadResult{}, err + } + + // Execute the request. + resp, err := c.do(req) + defer closeResponse(resp) + if err != nil { + return completeMultipartUploadResult{}, err + } + if resp != nil { + if resp.StatusCode != http.StatusOK { + return completeMultipartUploadResult{}, HTTPRespToErrorResponse(resp, bucketName, objectName) + } + } + // Decode completed multipart upload response on success. + completeMultipartUploadResult := completeMultipartUploadResult{} + err = xmlDecoder(resp.Body, &completeMultipartUploadResult) + if err != nil { + return completeMultipartUploadResult, err + } + return completeMultipartUploadResult, nil +} diff --git a/Godeps/_workspace/src/github.com/minio/minio-go/api-put-object-partial.go b/Godeps/_workspace/src/github.com/minio/minio-go/api-put-object-partial.go deleted file mode 100644 index 8c05d8858..000000000 --- a/Godeps/_workspace/src/github.com/minio/minio-go/api-put-object-partial.go +++ /dev/null @@ -1,378 +0,0 @@ -/* - * Minio Go Library for Amazon S3 Compatible Cloud Storage (C) 2015 Minio, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package minio - -import ( - "bytes" - "crypto/md5" - "crypto/sha256" - "errors" - "fmt" - "hash" - "io" - "io/ioutil" - "sort" -) - -// PutObjectPartial put object partial. -func (c Client) PutObjectPartial(bucketName, objectName string, data ReadAtCloser, size int64, contentType string) (n int64, err error) { - // Input validation. - if err := isValidBucketName(bucketName); err != nil { - return 0, err - } - if err := isValidObjectName(objectName); err != nil { - return 0, err - } - // Input size negative should return error. - if size < 0 { - return 0, ErrInvalidArgument("Input file size cannot be negative.") - } - // Input size bigger than 5TiB should fail. - if size > int64(maxMultipartPutObjectSize) { - return 0, ErrInvalidArgument("Input file size is bigger than the supported maximum of 5TiB.") - } - - // NOTE: Google Cloud Storage does not implement Amazon S3 Compatible multipart PUT. - // So we fall back to single PUT operation with the maximum limit of 5GiB. - if isGoogleEndpoint(c.endpointURL) { - if size > int64(maxSinglePutObjectSize) { - return 0, ErrorResponse{ - Code: "NotImplemented", - Message: fmt.Sprintf("Invalid Content-Length %d for file uploads to Google Cloud Storage.", size), - Key: objectName, - BucketName: bucketName, - } - } - // Do not compute MD5 for Google Cloud Storage. Uploads upto 5GiB in size. - n, err := c.putPartialNoChksum(bucketName, objectName, data, size, contentType) - return n, err - } - - // NOTE: S3 doesn't allow anonymous multipart requests. - if isAmazonEndpoint(c.endpointURL) && c.anonymous { - if size > int64(maxSinglePutObjectSize) { - return 0, ErrorResponse{ - Code: "NotImplemented", - Message: fmt.Sprintf("For anonymous requests Content-Length cannot be %d.", size), - Key: objectName, - BucketName: bucketName, - } - } - // Do not compute MD5 for anonymous requests to Amazon S3. Uploads upto 5GiB in size. - n, err := c.putPartialAnonymous(bucketName, objectName, data, size, contentType) - return n, err - } - - // Small file upload is initiated for uploads for input data size smaller than 5MiB. - if size < minimumPartSize { - n, err = c.putPartialSmallObject(bucketName, objectName, data, size, contentType) - return n, err - } - n, err = c.putPartialLargeObject(bucketName, objectName, data, size, contentType) - return n, err - -} - -// putNoChecksumPartial special function used Google Cloud Storage. This special function -// is used for Google Cloud Storage since Google's multipart API is not S3 compatible. -func (c Client) putPartialNoChksum(bucketName, objectName string, data ReadAtCloser, size int64, contentType string) (n int64, err error) { - // Input validation. - if err := isValidBucketName(bucketName); err != nil { - return 0, err - } - if err := isValidObjectName(objectName); err != nil { - return 0, err - } - if size > maxPartSize { - return 0, ErrEntityTooLarge(size, bucketName, objectName) - } - - // Create a new pipe to stage the reads. - reader, writer := io.Pipe() - - // readAtOffset to carry future offsets. - var readAtOffset int64 - - // readAt defaults to reading at 5MiB buffer. - readAtBuffer := make([]byte, 1024*1024*5) - - // Initiate a routine to start writing. - go func() { - for { - readAtSize, rerr := data.ReadAt(readAtBuffer, readAtOffset) - if rerr != nil { - if rerr != io.EOF { - writer.CloseWithError(rerr) - return - } - } - writeSize, werr := writer.Write(readAtBuffer[:readAtSize]) - if werr != nil { - writer.CloseWithError(werr) - return - } - if readAtSize != writeSize { - writer.CloseWithError(errors.New("Something really bad happened here. " + reportIssue)) - return - } - readAtOffset += int64(writeSize) - if rerr == io.EOF { - writer.Close() - return - } - } - }() - // For anonymous requests, we will not calculate sha256 and md5sum. - putObjData := putObjectData{ - MD5Sum: nil, - Sha256Sum: nil, - ReadCloser: reader, - Size: size, - ContentType: contentType, - } - // Execute put object. - st, err := c.putObject(bucketName, objectName, putObjData) - if err != nil { - return 0, err - } - if st.Size != size { - return 0, ErrUnexpectedEOF(st.Size, size, bucketName, objectName) - } - return size, nil -} - -// putAnonymousPartial is a special function for uploading content as anonymous request. -// This special function is necessary since Amazon S3 doesn't allow anonymous multipart uploads. -func (c Client) putPartialAnonymous(bucketName, objectName string, data ReadAtCloser, size int64, contentType string) (n int64, err error) { - // Input validation. - if err := isValidBucketName(bucketName); err != nil { - return 0, err - } - if err := isValidObjectName(objectName); err != nil { - return 0, err - } - return c.putPartialNoChksum(bucketName, objectName, data, size, contentType) -} - -// putSmallObjectPartial uploads files smaller than 5MiB. -func (c Client) putPartialSmallObject(bucketName, objectName string, data ReadAtCloser, size int64, contentType string) (n int64, err error) { - // Input validation. - if err := isValidBucketName(bucketName); err != nil { - return 0, err - } - if err := isValidObjectName(objectName); err != nil { - return 0, err - } - - // readAt defaults to reading at 5MiB buffer. - readAtBuffer := make([]byte, size) - readAtSize, err := data.ReadAt(readAtBuffer, 0) - if err != nil { - if err != io.EOF { - return 0, err - } - } - if int64(readAtSize) != size { - return 0, ErrUnexpectedEOF(int64(readAtSize), size, bucketName, objectName) - } - - // Construct a new PUT object metadata. - putObjData := putObjectData{ - MD5Sum: sumMD5(readAtBuffer), - Sha256Sum: sum256(readAtBuffer), - ReadCloser: ioutil.NopCloser(bytes.NewReader(readAtBuffer)), - Size: size, - ContentType: contentType, - } - // Single part use case, use putObject directly. - st, err := c.putObject(bucketName, objectName, putObjData) - if err != nil { - return 0, err - } - if st.Size != size { - return 0, ErrUnexpectedEOF(st.Size, size, bucketName, objectName) - } - return size, nil -} - -// putPartialLargeObject uploads files bigger than 5MiB. -func (c Client) putPartialLargeObject(bucketName, objectName string, data ReadAtCloser, size int64, contentType string) (n int64, err error) { - // Input validation. - if err := isValidBucketName(bucketName); err != nil { - return 0, err - } - if err := isValidObjectName(objectName); err != nil { - return 0, err - } - - // getUploadID for an object, initiates a new multipart request - // if it cannot find any previously partially uploaded object. - uploadID, err := c.getUploadID(bucketName, objectName, contentType) - if err != nil { - return 0, err - } - - // total data read and written to server. should be equal to 'size' at the end of the call. - var totalUploadedSize int64 - - // Complete multipart upload. - var completeMultipartUpload completeMultipartUpload - - // Fetch previously upload parts and save the total size. - partsInfo, err := c.listObjectParts(bucketName, objectName, uploadID) - if err != nil { - return 0, err - } - - // Previous maximum part size - var prevMaxPartSize int64 - // previous part number. - var prevPartNumber int - // Loop through all parts and calculate totalUploadedSize. - for _, partInfo := range partsInfo { - totalUploadedSize += partInfo.Size - // Choose the maximum part size. - if partInfo.Size >= prevMaxPartSize { - prevMaxPartSize = partInfo.Size - } - // Save previous part number. - prevPartNumber = partInfo.PartNumber - } - - // Calculate the optimal part size for a given file size. - partSize := optimalPartSize(size) - // If prevMaxPartSize is set use that. - if prevMaxPartSize != 0 { - partSize = prevMaxPartSize - } - - // MD5 and Sha256 hasher. - var hashMD5, hashSha256 hash.Hash - - // Part number always starts with prevPartNumber + 1. i.e The next part number. - partNumber := prevPartNumber + 1 - - // Loop through until EOF. - for totalUploadedSize < size { - // Initialize a new temporary file. - tmpFile, err := newTempFile("multiparts$-putobject-partial") - if err != nil { - return 0, err - } - - // Create a hash multiwriter. - hashMD5 = md5.New() - hashWriter := io.MultiWriter(hashMD5) - if c.signature.isV4() { - hashSha256 = sha256.New() - hashWriter = io.MultiWriter(hashMD5, hashSha256) - } - writer := io.MultiWriter(tmpFile, hashWriter) - - // totalUploadedSize is the current readAtOffset. - readAtOffset := totalUploadedSize - - // Read until partSize. - var totalReadPartSize int64 - - // readAt defaults to reading at 5MiB buffer. - readAtBuffer := make([]byte, optimalReadAtBufferSize) - - // Loop through until partSize. - for totalReadPartSize < partSize { - readAtSize, rerr := data.ReadAt(readAtBuffer, readAtOffset) - if rerr != nil { - if rerr != io.EOF { - return 0, rerr - } - } - writeSize, werr := writer.Write(readAtBuffer[:readAtSize]) - if werr != nil { - return 0, werr - } - if readAtSize != writeSize { - return 0, errors.New("Something really bad happened here. " + reportIssue) - } - readAtOffset += int64(writeSize) - totalReadPartSize += int64(writeSize) - if rerr == io.EOF { - break - } - } - - // Seek back to beginning of the temporary file. - if _, err := tmpFile.Seek(0, 0); err != nil { - return 0, err - } - - // Save all the part metadata. - prtData := partData{ - ReadCloser: tmpFile, - MD5Sum: hashMD5.Sum(nil), - Size: totalReadPartSize, - } - - // Signature version '4'. - if c.signature.isV4() { - prtData.Sha256Sum = hashSha256.Sum(nil) - } - - // Current part number to be uploaded. - prtData.Number = partNumber - - // execute upload part. - objPart, err := c.uploadPart(bucketName, objectName, uploadID, prtData) - if err != nil { - // Close the read closer. - prtData.ReadCloser.Close() - return totalUploadedSize, err - } - - // Save successfully uploaded size. - totalUploadedSize += prtData.Size - - // Save successfully uploaded part metadata. - partsInfo[prtData.Number] = objPart - - // Move to next part. - partNumber++ - } - - // If size is greater than zero verify totalUploaded. - // if totalUploaded is different than the input 'size', do not complete the request throw an error. - if totalUploadedSize != size { - return totalUploadedSize, ErrUnexpectedEOF(totalUploadedSize, size, bucketName, objectName) - } - - // Loop over uploaded parts to save them in a Parts array before completing the multipart request. - for _, part := range partsInfo { - var complPart completePart - complPart.ETag = part.ETag - complPart.PartNumber = part.PartNumber - completeMultipartUpload.Parts = append(completeMultipartUpload.Parts, complPart) - } - - // Sort all completed parts. - sort.Sort(completedParts(completeMultipartUpload.Parts)) - _, err = c.completeMultipartUpload(bucketName, objectName, uploadID, completeMultipartUpload) - if err != nil { - return totalUploadedSize, err - } - - // Return final size. - return totalUploadedSize, nil -} diff --git a/Godeps/_workspace/src/github.com/minio/minio-go/api-put-object-readat.go b/Godeps/_workspace/src/github.com/minio/minio-go/api-put-object-readat.go new file mode 100644 index 000000000..6d1b0e1fe --- /dev/null +++ b/Godeps/_workspace/src/github.com/minio/minio-go/api-put-object-readat.go @@ -0,0 +1,196 @@ +/* + * Minio Go Library for Amazon S3 Compatible Cloud Storage (C) 2015 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package minio + +import ( + "crypto/md5" + "crypto/sha256" + "errors" + "hash" + "io" + "sort" +) + +// putObjectMultipartFromReadAt - Uploads files bigger than 5MiB. Supports reader +// of type which implements io.ReaderAt interface (ReadAt method). +// +// NOTE: This function is meant to be used for all readers which +// implement io.ReaderAt which allows us for resuming multipart +// uploads but reading at an offset, which would avoid re-read the +// data which was already uploaded. Internally this function uses +// temporary files for staging all the data, these temporary files are +// cleaned automatically when the caller i.e http client closes the +// stream after uploading all the contents successfully. +func (c Client) putObjectMultipartFromReadAt(bucketName, objectName string, reader io.ReaderAt, size int64, contentType string) (n int64, err error) { + // Input validation. + if err := isValidBucketName(bucketName); err != nil { + return 0, err + } + if err := isValidObjectName(objectName); err != nil { + return 0, err + } + + // Get upload id for an object, initiates a new multipart request + // if it cannot find any previously partially uploaded object. + uploadID, err := c.getUploadID(bucketName, objectName, contentType) + if err != nil { + return 0, err + } + + // Total data read and written to server. should be equal to 'size' at the end of the call. + var totalUploadedSize int64 + + // Complete multipart upload. + var completeMultipartUpload completeMultipartUpload + + // Fetch previously upload parts. + partsInfo, err := c.listObjectParts(bucketName, objectName, uploadID) + if err != nil { + return 0, err + } + + // Previous maximum part size + var prevMaxPartSize int64 + // Previous part number. + var prevPartNumber int + // Loop through all parts and calculate totalUploadedSize. + for _, partInfo := range partsInfo { + totalUploadedSize += partInfo.Size + // Choose the maximum part size. + if partInfo.Size >= prevMaxPartSize { + prevMaxPartSize = partInfo.Size + } + // Save previous part number. + prevPartNumber = partInfo.PartNumber + } + + // Calculate the optimal part size for a given file size. + partSize := optimalPartSize(size) + // If prevMaxPartSize is set use that. + if prevMaxPartSize != 0 { + partSize = prevMaxPartSize + } + + // MD5 and SHA256 hasher. + var hashMD5, hashSHA256 hash.Hash + + // Part number always starts with prevPartNumber + 1. i.e The next part number. + partNumber := prevPartNumber + 1 + + // Upload each part until totalUploadedSize reaches input reader size. + for totalUploadedSize < size { + // Initialize a new temporary file. + tmpFile, err := newTempFile("multiparts$-putobject-partial") + if err != nil { + return 0, err + } + + // Create a hash multiwriter. + hashMD5 = md5.New() + hashWriter := io.MultiWriter(hashMD5) + if c.signature.isV4() { + hashSHA256 = sha256.New() + hashWriter = io.MultiWriter(hashMD5, hashSHA256) + } + writer := io.MultiWriter(tmpFile, hashWriter) + + // Choose totalUploadedSize as the current readAtOffset. + readAtOffset := totalUploadedSize + + // Read until partSize. + var totalReadPartSize int64 + + // ReadAt defaults to reading at 5MiB buffer. + readAtBuffer := make([]byte, optimalReadAtBufferSize) + + // Following block reads data at an offset from the input + // reader and copies data to into local temporary file. + // Temporary file data is limited to the partSize. + for totalReadPartSize < partSize { + readAtSize, rerr := reader.ReadAt(readAtBuffer, readAtOffset) + if rerr != nil { + if rerr != io.EOF { + return 0, rerr + } + } + writeSize, werr := writer.Write(readAtBuffer[:readAtSize]) + if werr != nil { + return 0, werr + } + if readAtSize != writeSize { + return 0, errors.New("Something really bad happened here. " + reportIssue) + } + readAtOffset += int64(writeSize) + totalReadPartSize += int64(writeSize) + if rerr == io.EOF { + break + } + } + + // Seek back to beginning of the temporary file. + if _, err := tmpFile.Seek(0, 0); err != nil { + return 0, err + } + + var md5Sum, sha256Sum []byte + md5Sum = hashMD5.Sum(nil) + // Signature version '4'. + if c.signature.isV4() { + sha256Sum = hashSHA256.Sum(nil) + } + + // Proceed to upload the part. + objPart, err := c.uploadPart(bucketName, objectName, uploadID, tmpFile, partNumber, md5Sum, sha256Sum, totalReadPartSize) + if err != nil { + // Close the read closer. + tmpFile.Close() + return totalUploadedSize, err + } + + // Save successfully uploaded size. + totalUploadedSize += totalReadPartSize + + // Save successfully uploaded part metadata. + partsInfo[partNumber] = objPart + + // Move to next part. + partNumber++ + } + + // Verify if we uploaded all the data. + if totalUploadedSize != size { + return totalUploadedSize, ErrUnexpectedEOF(totalUploadedSize, size, bucketName, objectName) + } + + // Loop over uploaded parts to save them in a Parts array before completing the multipart request. + for _, part := range partsInfo { + var complPart completePart + complPart.ETag = part.ETag + complPart.PartNumber = part.PartNumber + completeMultipartUpload.Parts = append(completeMultipartUpload.Parts, complPart) + } + + // Sort all completed parts. + sort.Sort(completedParts(completeMultipartUpload.Parts)) + _, err = c.completeMultipartUpload(bucketName, objectName, uploadID, completeMultipartUpload) + if err != nil { + return totalUploadedSize, err + } + + // Return final size. + return totalUploadedSize, nil +} diff --git a/Godeps/_workspace/src/github.com/minio/minio-go/api-put-object.go b/Godeps/_workspace/src/github.com/minio/minio-go/api-put-object.go index 563856bae..02f27642f 100644 --- a/Godeps/_workspace/src/github.com/minio/minio-go/api-put-object.go +++ b/Godeps/_workspace/src/github.com/minio/minio-go/api-put-object.go @@ -18,21 +18,43 @@ package minio import ( "bytes" - "crypto/md5" - "crypto/sha256" - "encoding/hex" - "encoding/xml" - "fmt" - "hash" "io" "io/ioutil" "net/http" - "net/url" - "sort" - "strconv" + "os" "strings" ) +// getReaderSize gets the size of the underlying reader, if possible. +func getReaderSize(reader io.Reader) (size int64, err error) { + size = -1 + if reader != nil { + switch v := reader.(type) { + case *bytes.Buffer: + size = int64(v.Len()) + case *bytes.Reader: + size = int64(v.Len()) + case *strings.Reader: + size = int64(v.Len()) + case *os.File: + var st os.FileInfo + st, err = v.Stat() + if err != nil { + return 0, err + } + size = st.Size() + case *Object: + var st ObjectInfo + st, err = v.Stat() + if err != nil { + return 0, err + } + size = st.Size + } + } + return size, nil +} + // completedParts is a collection of parts sortable by their part numbers. // used for sorting the uploaded parts before completing the multipart request. type completedParts []completePart @@ -54,7 +76,7 @@ func (a completedParts) Less(i, j int) bool { return a[i].PartNumber < a[j].Part // So we fall back to single PUT operation with the maximum limit of 5GiB. // // NOTE: For anonymous requests Amazon S3 doesn't allow multipart upload. So we fall back to single PUT operation. -func (c Client) PutObject(bucketName, objectName string, data io.Reader, size int64, contentType string) (n int64, err error) { +func (c Client) PutObject(bucketName, objectName string, reader io.Reader, contentType string) (n int64, err error) { // Input validation. if err := isValidBucketName(bucketName); err != nil { return 0, err @@ -63,6 +85,17 @@ func (c Client) PutObject(bucketName, objectName string, data io.Reader, size in return 0, err } + // get reader size. + size, err := getReaderSize(reader) + if err != nil { + return 0, err + } + + // Check for largest object size allowed. + if size > int64(maxMultipartPutObjectSize) { + return 0, ErrEntityTooLarge(size, bucketName, objectName) + } + // NOTE: Google Cloud Storage does not implement Amazon S3 Compatible multipart PUT. // So we fall back to single PUT operation with the maximum limit of 5GiB. if isGoogleEndpoint(c.endpointURL) { @@ -74,35 +107,56 @@ func (c Client) PutObject(bucketName, objectName string, data io.Reader, size in BucketName: bucketName, } } + if size > maxSinglePutObjectSize { + return 0, ErrEntityTooLarge(size, bucketName, objectName) + } // Do not compute MD5 for Google Cloud Storage. Uploads upto 5GiB in size. - return c.putNoChecksum(bucketName, objectName, data, size, contentType) + return c.putObjectNoChecksum(bucketName, objectName, reader, size, contentType) } // NOTE: S3 doesn't allow anonymous multipart requests. if isAmazonEndpoint(c.endpointURL) && c.anonymous { - if size <= -1 || size > int64(maxSinglePutObjectSize) { + if size <= -1 { return 0, ErrorResponse{ Code: "NotImplemented", - Message: fmt.Sprintf("For anonymous requests Content-Length cannot be %d.", size), + Message: "Content-Length cannot be negative for anonymous requests.", Key: objectName, BucketName: bucketName, } } + if size > maxSinglePutObjectSize { + return 0, ErrEntityTooLarge(size, bucketName, objectName) + } // Do not compute MD5 for anonymous requests to Amazon S3. Uploads upto 5GiB in size. - return c.putAnonymous(bucketName, objectName, data, size, contentType) + return c.putObjectNoChecksum(bucketName, objectName, reader, size, contentType) } - // Large file upload is initiated for uploads for input data size - // if its greater than 5MiB or data size is negative. - if size >= minimumPartSize || size < 0 { - return c.putLargeObject(bucketName, objectName, data, size, contentType) + // putSmall object. + if size < minimumPartSize && size > 0 { + return c.putObjectSingle(bucketName, objectName, reader, size, contentType) } - return c.putSmallObject(bucketName, objectName, data, size, contentType) + // For all sizes greater than 5MiB do multipart. + n, err = c.putObjectMultipart(bucketName, objectName, reader, size, contentType) + if err != nil { + errResp := ToErrorResponse(err) + // Verify if multipart functionality is not available, if not + // fall back to single PutObject operation. + if errResp.Code == "NotImplemented" { + // Verify if size of reader is greater than '5GiB'. + if size > maxSinglePutObjectSize { + return 0, ErrEntityTooLarge(size, bucketName, objectName) + } + // Fall back to uploading as single PutObject operation. + return c.putObjectSingle(bucketName, objectName, reader, size, contentType) + } + return n, err + } + return n, nil } -// putNoChecksum special function used Google Cloud Storage. This special function +// putObjectNoChecksum special function used Google Cloud Storage. This special function // is used for Google Cloud Storage since Google's multipart API is not S3 compatible. -func (c Client) putNoChecksum(bucketName, objectName string, data io.Reader, size int64, contentType string) (n int64, err error) { +func (c Client) putObjectNoChecksum(bucketName, objectName string, reader io.Reader, size int64, contentType string) (n int64, err error) { // Input validation. if err := isValidBucketName(bucketName); err != nil { return 0, err @@ -110,19 +164,12 @@ func (c Client) putNoChecksum(bucketName, objectName string, data io.Reader, siz if err := isValidObjectName(objectName); err != nil { return 0, err } - if size > maxPartSize { + if size > maxSinglePutObjectSize { return 0, ErrEntityTooLarge(size, bucketName, objectName) } - // For anonymous requests, we will not calculate sha256 and md5sum. - putObjData := putObjectData{ - MD5Sum: nil, - Sha256Sum: nil, - ReadCloser: ioutil.NopCloser(data), - Size: size, - ContentType: contentType, - } + // This function does not calculate sha256 and md5sum for payload. // Execute put object. - st, err := c.putObject(bucketName, objectName, putObjData) + st, err := c.putObjectDo(bucketName, objectName, ioutil.NopCloser(reader), nil, nil, size, contentType) if err != nil { return 0, err } @@ -132,10 +179,9 @@ func (c Client) putNoChecksum(bucketName, objectName string, data io.Reader, siz return size, nil } -// putAnonymous is a special function for uploading content as anonymous request. -// This special function is necessary since Amazon S3 doesn't allow anonymous -// multipart uploads. -func (c Client) putAnonymous(bucketName, objectName string, data io.Reader, size int64, contentType string) (n int64, err error) { +// putObjectSingle is a special function for uploading single put object request. +// This special function is used as a fallback when multipart upload fails. +func (c Client) putObjectSingle(bucketName, objectName string, reader io.Reader, size int64, contentType string) (n int64, err error) { // Input validation. if err := isValidBucketName(bucketName); err != nil { return 0, err @@ -143,431 +189,96 @@ func (c Client) putAnonymous(bucketName, objectName string, data io.Reader, size if err := isValidObjectName(objectName); err != nil { return 0, err } - return c.putNoChecksum(bucketName, objectName, data, size, contentType) -} - -// putSmallObject uploads files smaller than 5 mega bytes. -func (c Client) putSmallObject(bucketName, objectName string, data io.Reader, size int64, contentType string) (n int64, err error) { - // Input validation. - if err := isValidBucketName(bucketName); err != nil { - return 0, err + if size > maxSinglePutObjectSize { + return 0, ErrEntityTooLarge(size, bucketName, objectName) } - if err := isValidObjectName(objectName); err != nil { - return 0, err + // If size is a stream, upload upto 5GiB. + if size <= -1 { + size = maxSinglePutObjectSize } - // Read input data fully into buffer. - dataBytes, err := ioutil.ReadAll(data) + // Initialize a new temporary file. + tmpFile, err := newTempFile("single$-putobject-single") if err != nil { return 0, err } - if int64(len(dataBytes)) != size { - return 0, ErrUnexpectedEOF(int64(len(dataBytes)), size, bucketName, objectName) - } - // Construct a new PUT object metadata. - putObjData := putObjectData{ - MD5Sum: sumMD5(dataBytes), - Sha256Sum: sum256(dataBytes), - ReadCloser: ioutil.NopCloser(bytes.NewReader(dataBytes)), - Size: size, - ContentType: contentType, - } - // Single part use case, use putObject directly. - st, err := c.putObject(bucketName, objectName, putObjData) + md5Sum, sha256Sum, size, err := c.hashCopyN(tmpFile, reader, size) if err != nil { - return 0, err - } - if st.Size != size { - return 0, ErrUnexpectedEOF(st.Size, size, bucketName, objectName) - } - return size, nil -} - -// hashCopy - calculates Md5sum and Sha256sum for upto partSize amount of bytes. -func (c Client) hashCopy(writer io.ReadWriteSeeker, data io.Reader, partSize int64) (md5Sum, sha256Sum []byte, size int64, err error) { - // MD5 and Sha256 hasher. - var hashMD5, hashSha256 hash.Hash - // MD5 and Sha256 hasher. - hashMD5 = md5.New() - hashWriter := io.MultiWriter(writer, hashMD5) - if c.signature.isV4() { - hashSha256 = sha256.New() - hashWriter = io.MultiWriter(writer, hashMD5, hashSha256) - } - - // Copies to input at writer. - size, err = io.CopyN(hashWriter, data, partSize) - if err != nil { - // If not EOF return error right here. if err != io.EOF { - return nil, nil, 0, err - } - } - - // Seek back to beginning of input, any error fail right here. - if _, err := writer.Seek(0, 0); err != nil { - return nil, nil, 0, err - } - - // Finalize md5shum and sha256 sum. - md5Sum = hashMD5.Sum(nil) - if c.signature.isV4() { - sha256Sum = hashSha256.Sum(nil) - } - return md5Sum, sha256Sum, size, err -} - -// putLargeObject uploads files bigger than 5 mega bytes. -func (c Client) putLargeObject(bucketName, objectName string, data io.Reader, size int64, contentType string) (n int64, err error) { - // Input validation. - if err := isValidBucketName(bucketName); err != nil { - return 0, err - } - if err := isValidObjectName(objectName); err != nil { - return 0, err - } - - // getUploadID for an object, initiates a new multipart request - // if it cannot find any previously partially uploaded object. - uploadID, err := c.getUploadID(bucketName, objectName, contentType) - if err != nil { - return 0, err - } - - // total data read and written to server. should be equal to 'size' at the end of the call. - var totalUploadedSize int64 - - // Complete multipart upload. - var completeMultipartUpload completeMultipartUpload - - // Fetch previously upload parts and save the total size. - partsInfo, err := c.listObjectParts(bucketName, objectName, uploadID) - if err != nil { - return 0, err - } - // Previous maximum part size - var prevMaxPartSize int64 - // Loop through all parts and calculate totalUploadedSize. - for _, partInfo := range partsInfo { - // Choose the maximum part size. - if partInfo.Size >= prevMaxPartSize { - prevMaxPartSize = partInfo.Size - } - } - - // Calculate the optimal part size for a given size. - partSize := optimalPartSize(size) - // If prevMaxPartSize is set use that. - if prevMaxPartSize != 0 { - partSize = prevMaxPartSize - } - - // Part number always starts with '0'. - partNumber := 0 - - // Loop through until EOF. - for { - // Increment part number. - partNumber++ - - // Initialize a new temporary file. - tmpFile, err := newTempFile("multiparts$-putobject") - if err != nil { return 0, err } - - // Calculates MD5 and Sha256 sum while copying partSize bytes into tmpFile. - md5Sum, sha256Sum, size, rErr := c.hashCopy(tmpFile, data, partSize) - if rErr != nil { - if rErr != io.EOF { - return 0, rErr - } - } - - // Save all the part metadata. - prtData := partData{ - ReadCloser: tmpFile, - Size: size, - MD5Sum: md5Sum, - Sha256Sum: sha256Sum, - Number: partNumber, // Current part number to be uploaded. - } - - // If part not uploaded proceed to upload. - if !isPartUploaded(objectPart{ - ETag: hex.EncodeToString(prtData.MD5Sum), - PartNumber: partNumber, - }, partsInfo) { - // execute upload part. - objPart, err := c.uploadPart(bucketName, objectName, uploadID, prtData) - if err != nil { - // Close the read closer. - prtData.ReadCloser.Close() - return 0, err - } - // Save successfully uploaded part metadata. - partsInfo[prtData.Number] = objPart - } - - // Close the read closer. - prtData.ReadCloser.Close() - - // If read error was an EOF, break out of the loop. - if rErr == io.EOF { - break - } } - - // Loop over uploaded parts to save them in a Parts array before completing the multipart request. - for _, part := range partsInfo { - var complPart completePart - complPart.ETag = part.ETag - complPart.PartNumber = part.PartNumber - completeMultipartUpload.Parts = append(completeMultipartUpload.Parts, complPart) - // Save successfully uploaded size. - totalUploadedSize += part.Size - } - - // If size is greater than zero verify totalUploadedSize. if totalUploadedSize is - // different than the input 'size', do not complete the request throw an error. - if size > 0 { - if totalUploadedSize != size { - return totalUploadedSize, ErrUnexpectedEOF(totalUploadedSize, size, bucketName, objectName) - } - } - - // If partNumber is different than total list of parts, error out. - if partNumber != len(completeMultipartUpload.Parts) { - return totalUploadedSize, ErrInvalidParts(partNumber, len(completeMultipartUpload.Parts)) - } - - // Sort all completed parts. - sort.Sort(completedParts(completeMultipartUpload.Parts)) - _, err = c.completeMultipartUpload(bucketName, objectName, uploadID, completeMultipartUpload) + // Execute put object. + st, err := c.putObjectDo(bucketName, objectName, tmpFile, md5Sum, sha256Sum, size, contentType) if err != nil { - return totalUploadedSize, err + return 0, err } - - // Return final size. - return totalUploadedSize, nil + if st.Size != size { + return 0, ErrUnexpectedEOF(st.Size, size, bucketName, objectName) + } + return size, nil } -// putObject - add an object to a bucket. +// putObjectDo - executes the put object http operation. // NOTE: You must have WRITE permissions on a bucket to add an object to it. -func (c Client) putObject(bucketName, objectName string, putObjData putObjectData) (ObjectStat, error) { +func (c Client) putObjectDo(bucketName, objectName string, reader io.ReadCloser, md5Sum []byte, sha256Sum []byte, size int64, contentType string) (ObjectInfo, error) { // Input validation. if err := isValidBucketName(bucketName); err != nil { - return ObjectStat{}, err + return ObjectInfo{}, err } if err := isValidObjectName(objectName); err != nil { - return ObjectStat{}, err + return ObjectInfo{}, err } - if strings.TrimSpace(putObjData.ContentType) == "" { - putObjData.ContentType = "application/octet-stream" + if size <= -1 { + return ObjectInfo{}, ErrEntityTooSmall(size, bucketName, objectName) + } + + if size > maxSinglePutObjectSize { + return ObjectInfo{}, ErrEntityTooLarge(size, bucketName, objectName) + } + + if strings.TrimSpace(contentType) == "" { + contentType = "application/octet-stream" } // Set headers. customHeader := make(http.Header) - customHeader.Set("Content-Type", putObjData.ContentType) + customHeader.Set("Content-Type", contentType) // Populate request metadata. reqMetadata := requestMetadata{ bucketName: bucketName, objectName: objectName, customHeader: customHeader, - contentBody: putObjData.ReadCloser, - contentLength: putObjData.Size, - contentSha256Bytes: putObjData.Sha256Sum, - contentMD5Bytes: putObjData.MD5Sum, + contentBody: reader, + contentLength: size, + contentMD5Bytes: md5Sum, + contentSHA256Bytes: sha256Sum, } // Initiate new request. req, err := c.newRequest("PUT", reqMetadata) if err != nil { - return ObjectStat{}, err + return ObjectInfo{}, err } // Execute the request. resp, err := c.do(req) defer closeResponse(resp) if err != nil { - return ObjectStat{}, err + return ObjectInfo{}, err } if resp != nil { if resp.StatusCode != http.StatusOK { - return ObjectStat{}, HTTPRespToErrorResponse(resp, bucketName, objectName) + return ObjectInfo{}, HTTPRespToErrorResponse(resp, bucketName, objectName) } } - var metadata ObjectStat + var metadata ObjectInfo // Trim off the odd double quotes from ETag in the beginning and end. metadata.ETag = strings.TrimPrefix(resp.Header.Get("ETag"), "\"") metadata.ETag = strings.TrimSuffix(metadata.ETag, "\"") // A success here means data was written to server successfully. - metadata.Size = putObjData.Size + metadata.Size = size // Return here. return metadata, nil } - -// initiateMultipartUpload initiates a multipart upload and returns an upload ID. -func (c Client) initiateMultipartUpload(bucketName, objectName, contentType string) (initiateMultipartUploadResult, error) { - // Input validation. - if err := isValidBucketName(bucketName); err != nil { - return initiateMultipartUploadResult{}, err - } - if err := isValidObjectName(objectName); err != nil { - return initiateMultipartUploadResult{}, err - } - - // Initialize url queries. - urlValues := make(url.Values) - urlValues.Set("uploads", "") - - if contentType == "" { - contentType = "application/octet-stream" - } - - // set ContentType header. - customHeader := make(http.Header) - customHeader.Set("Content-Type", contentType) - - reqMetadata := requestMetadata{ - bucketName: bucketName, - objectName: objectName, - queryValues: urlValues, - customHeader: customHeader, - } - - // Instantiate the request. - req, err := c.newRequest("POST", reqMetadata) - if err != nil { - return initiateMultipartUploadResult{}, err - } - // Execute the request. - resp, err := c.do(req) - defer closeResponse(resp) - if err != nil { - return initiateMultipartUploadResult{}, err - } - if resp != nil { - if resp.StatusCode != http.StatusOK { - return initiateMultipartUploadResult{}, HTTPRespToErrorResponse(resp, bucketName, objectName) - } - } - // Decode xml initiate multipart. - initiateMultipartUploadResult := initiateMultipartUploadResult{} - err = xmlDecoder(resp.Body, &initiateMultipartUploadResult) - if err != nil { - return initiateMultipartUploadResult, err - } - return initiateMultipartUploadResult, nil -} - -// uploadPart uploads a part in a multipart upload. -func (c Client) uploadPart(bucketName, objectName, uploadID string, uploadingPart partData) (objectPart, error) { - // Input validation. - if err := isValidBucketName(bucketName); err != nil { - return objectPart{}, err - } - if err := isValidObjectName(objectName); err != nil { - return objectPart{}, err - } - - // Get resources properly escaped and lined up before using them in http request. - urlValues := make(url.Values) - // Set part number. - urlValues.Set("partNumber", strconv.Itoa(uploadingPart.Number)) - // Set upload id. - urlValues.Set("uploadId", uploadID) - - reqMetadata := requestMetadata{ - bucketName: bucketName, - objectName: objectName, - queryValues: urlValues, - contentBody: uploadingPart.ReadCloser, - contentLength: uploadingPart.Size, - contentSha256Bytes: uploadingPart.Sha256Sum, - contentMD5Bytes: uploadingPart.MD5Sum, - } - - // Instantiate a request. - req, err := c.newRequest("PUT", reqMetadata) - if err != nil { - return objectPart{}, err - } - // Execute the request. - resp, err := c.do(req) - defer closeResponse(resp) - if err != nil { - return objectPart{}, err - } - if resp != nil { - if resp.StatusCode != http.StatusOK { - return objectPart{}, HTTPRespToErrorResponse(resp, bucketName, objectName) - } - } - // Once successfully uploaded, return completed part. - objPart := objectPart{} - objPart.Size = uploadingPart.Size - objPart.PartNumber = uploadingPart.Number - // Trim off the odd double quotes from ETag in the beginning and end. - objPart.ETag = strings.TrimPrefix(resp.Header.Get("ETag"), "\"") - objPart.ETag = strings.TrimSuffix(objPart.ETag, "\"") - return objPart, nil -} - -// completeMultipartUpload completes a multipart upload by assembling previously uploaded parts. -func (c Client) completeMultipartUpload(bucketName, objectName, uploadID string, complete completeMultipartUpload) (completeMultipartUploadResult, error) { - // Input validation. - if err := isValidBucketName(bucketName); err != nil { - return completeMultipartUploadResult{}, err - } - if err := isValidObjectName(objectName); err != nil { - return completeMultipartUploadResult{}, err - } - - // Initialize url queries. - urlValues := make(url.Values) - urlValues.Set("uploadId", uploadID) - - // Marshal complete multipart body. - completeMultipartUploadBytes, err := xml.Marshal(complete) - if err != nil { - return completeMultipartUploadResult{}, err - } - - // Instantiate all the complete multipart buffer. - completeMultipartUploadBuffer := bytes.NewBuffer(completeMultipartUploadBytes) - reqMetadata := requestMetadata{ - bucketName: bucketName, - objectName: objectName, - queryValues: urlValues, - contentBody: ioutil.NopCloser(completeMultipartUploadBuffer), - contentLength: int64(completeMultipartUploadBuffer.Len()), - contentSha256Bytes: sum256(completeMultipartUploadBuffer.Bytes()), - } - - // Instantiate the request. - req, err := c.newRequest("POST", reqMetadata) - if err != nil { - return completeMultipartUploadResult{}, err - } - - // Execute the request. - resp, err := c.do(req) - defer closeResponse(resp) - if err != nil { - return completeMultipartUploadResult{}, err - } - if resp != nil { - if resp.StatusCode != http.StatusOK { - return completeMultipartUploadResult{}, HTTPRespToErrorResponse(resp, bucketName, objectName) - } - } - // If successful response, decode the body. - completeMultipartUploadResult := completeMultipartUploadResult{} - err = xmlDecoder(resp.Body, &completeMultipartUploadResult) - if err != nil { - return completeMultipartUploadResult, err - } - return completeMultipartUploadResult, nil -} diff --git a/Godeps/_workspace/src/github.com/minio/minio-go/api-remove.go b/Godeps/_workspace/src/github.com/minio/minio-go/api-remove.go index 0e1abc2e3..1ac420782 100644 --- a/Godeps/_workspace/src/github.com/minio/minio-go/api-remove.go +++ b/Godeps/_workspace/src/github.com/minio/minio-go/api-remove.go @@ -26,15 +26,18 @@ import ( // All objects (including all object versions and delete markers). // in the bucket must be deleted before successfully attempting this request. func (c Client) RemoveBucket(bucketName string) error { + // Input validation. if err := isValidBucketName(bucketName); err != nil { return err } + // Instantiate a new request. req, err := c.newRequest("DELETE", requestMetadata{ bucketName: bucketName, }) if err != nil { return err } + // Initiate the request. resp, err := c.do(req) defer closeResponse(resp) if err != nil { @@ -54,12 +57,14 @@ func (c Client) RemoveBucket(bucketName string) error { // RemoveObject remove an object from a bucket. func (c Client) RemoveObject(bucketName, objectName string) error { + // Input validation. if err := isValidBucketName(bucketName); err != nil { return err } if err := isValidObjectName(objectName); err != nil { return err } + // Instantiate the request. req, err := c.newRequest("DELETE", requestMetadata{ bucketName: bucketName, objectName: objectName, @@ -67,6 +72,7 @@ func (c Client) RemoveObject(bucketName, objectName string) error { if err != nil { return err } + // Initiate the request. resp, err := c.do(req) defer closeResponse(resp) if err != nil { @@ -81,42 +87,32 @@ func (c Client) RemoveObject(bucketName, objectName string) error { // RemoveIncompleteUpload aborts an partially uploaded object. // Requires explicit authentication, no anonymous requests are allowed for multipart API. func (c Client) RemoveIncompleteUpload(bucketName, objectName string) error { - // Validate input arguments. + // Input validation. if err := isValidBucketName(bucketName); err != nil { return err } if err := isValidObjectName(objectName); err != nil { return err } - errorCh := make(chan error) - go func(errorCh chan<- error) { - defer close(errorCh) - // Find multipart upload id of the object. - uploadID, err := c.findUploadID(bucketName, objectName) - if err != nil { - errorCh <- err - return - } - if uploadID != "" { - // If uploadID is not an empty string, initiate the request. - err := c.abortMultipartUpload(bucketName, objectName, uploadID) - if err != nil { - errorCh <- err - return - } - return - } - }(errorCh) - err, ok := <-errorCh - if ok && err != nil { + // Find multipart upload id of the object to be aborted. + uploadID, err := c.findUploadID(bucketName, objectName) + if err != nil { return err } + if uploadID != "" { + // Upload id found, abort the incomplete multipart upload. + err := c.abortMultipartUpload(bucketName, objectName, uploadID) + if err != nil { + return err + } + } return nil } -// abortMultipartUpload aborts a multipart upload for the given uploadID, all parts are deleted. +// abortMultipartUpload aborts a multipart upload for the given +// uploadID, all previously uploaded parts are deleted. func (c Client) abortMultipartUpload(bucketName, objectName, uploadID string) error { - // Validate input arguments. + // Input validation. if err := isValidBucketName(bucketName); err != nil { return err } @@ -138,7 +134,7 @@ func (c Client) abortMultipartUpload(bucketName, objectName, uploadID string) er return err } - // execute the request. + // Initiate the request. resp, err := c.do(req) defer closeResponse(resp) if err != nil { @@ -146,11 +142,12 @@ func (c Client) abortMultipartUpload(bucketName, objectName, uploadID string) er } if resp != nil { if resp.StatusCode != http.StatusNoContent { - // Abort has no response body, handle it. + // Abort has no response body, handle it for any errors. var errorResponse ErrorResponse switch resp.StatusCode { case http.StatusNotFound: - // This is needed specifically for Abort and it cannot be converged. + // This is needed specifically for abort and it cannot + // be converged into default case. errorResponse = ErrorResponse{ Code: "NoSuchUpload", Message: "The specified multipart upload does not exist.", diff --git a/Godeps/_workspace/src/github.com/minio/minio-go/api-s3-definitions.go b/Godeps/_workspace/src/github.com/minio/minio-go/api-s3-definitions.go index 61931b0b3..de562e475 100644 --- a/Godeps/_workspace/src/github.com/minio/minio-go/api-s3-definitions.go +++ b/Godeps/_workspace/src/github.com/minio/minio-go/api-s3-definitions.go @@ -21,31 +21,34 @@ import ( "time" ) -// listAllMyBucketsResult container for listBuckets response +// listAllMyBucketsResult container for listBuckets response. type listAllMyBucketsResult struct { // Container for one or more buckets. Buckets struct { - Bucket []BucketStat + Bucket []BucketInfo } Owner owner } -// owner container for bucket owner information +// owner container for bucket owner information. type owner struct { DisplayName string ID string } -// commonPrefix container for prefix response +// commonPrefix container for prefix response. type commonPrefix struct { Prefix string } -// listBucketResult container for listObjects response +// listBucketResult container for listObjects response. type listBucketResult struct { - CommonPrefixes []commonPrefix // A response can contain CommonPrefixes only if you have specified a delimiter - Contents []ObjectStat // Metadata about each object returned - Delimiter string + // A response can contain CommonPrefixes only if you have + // specified a delimiter. + CommonPrefixes []commonPrefix + // Metadata about each object returned. + Contents []ObjectInfo + Delimiter string // Encoding type used to encode object keys in the response. EncodingType string @@ -57,13 +60,15 @@ type listBucketResult struct { MaxKeys int64 Name string - // When response is truncated (the IsTruncated element value in the response - // is true), you can use the key name in this field as marker in the subsequent - // request to get next set of objects. Object storage lists objects in alphabetical - // order Note: This element is returned only if you have delimiter request parameter - // specified. If response does not include the NextMaker and it is truncated, - // you can use the value of the last Key in the response as the marker in the - // subsequent request to get the next set of object keys. + // When response is truncated (the IsTruncated element value in + // the response is true), you can use the key name in this field + // as marker in the subsequent request to get next set of objects. + // Object storage lists objects in alphabetical order Note: This + // element is returned only if you have delimiter request + // parameter specified. If response does not include the NextMaker + // and it is truncated, you can use the value of the last Key in + // the response as the marker in the subsequent request to get the + // next set of object keys. NextMarker string Prefix string } @@ -78,19 +83,20 @@ type listMultipartUploadsResult struct { EncodingType string MaxUploads int64 IsTruncated bool - Uploads []ObjectMultipartStat `xml:"Upload"` + Uploads []ObjectMultipartInfo `xml:"Upload"` Prefix string Delimiter string - CommonPrefixes []commonPrefix // A response can contain CommonPrefixes only if you specify a delimiter + // A response can contain CommonPrefixes only if you specify a delimiter. + CommonPrefixes []commonPrefix } -// initiator container for who initiated multipart upload +// initiator container for who initiated multipart upload. type initiator struct { ID string DisplayName string } -// objectPart container for particular part of an object +// objectPart container for particular part of an object. type objectPart struct { // Part number identifies the part. PartNumber int @@ -98,7 +104,8 @@ type objectPart struct { // Date and time the part was uploaded. LastModified time.Time - // Entity tag returned when the part was uploaded, usually md5sum of the part + // Entity tag returned when the part was uploaded, usually md5sum + // of the part. ETag string // Size of the uploaded part data. @@ -126,14 +133,16 @@ type listObjectPartsResult struct { EncodingType string } -// initiateMultipartUploadResult container for InitiateMultiPartUpload response. +// initiateMultipartUploadResult container for InitiateMultiPartUpload +// response. type initiateMultipartUploadResult struct { Bucket string Key string UploadID string `xml:"UploadId"` } -// completeMultipartUploadResult container for completed multipart upload response. +// completeMultipartUploadResult container for completed multipart +// upload response. type completeMultipartUploadResult struct { Location string Bucket string @@ -141,7 +150,8 @@ type completeMultipartUploadResult struct { ETag string } -// completePart sub container lists individual part numbers and their md5sum, part of completeMultipartUpload. +// completePart sub container lists individual part numbers and their +// md5sum, part of completeMultipartUpload. type completePart struct { XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Part" json:"-"` @@ -150,13 +160,13 @@ type completePart struct { ETag string } -// completeMultipartUpload container for completing multipart upload +// completeMultipartUpload container for completing multipart upload. type completeMultipartUpload struct { XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ CompleteMultipartUpload" json:"-"` Parts []completePart `xml:"Part"` } -// createBucketConfiguration container for bucket configuration +// createBucketConfiguration container for bucket configuration. type createBucketConfiguration struct { XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ CreateBucketConfiguration" json:"-"` Location string `xml:"LocationConstraint"` @@ -164,7 +174,8 @@ type createBucketConfiguration struct { // grant container for the grantee and his or her permissions. type grant struct { - // grantee container for DisplayName and ID of the person being granted permissions. + // grantee container for DisplayName and ID of the person being + // granted permissions. Grantee struct { ID string DisplayName string @@ -175,7 +186,8 @@ type grant struct { Permission string } -// accessControlPolicy contains the elements providing ACL permissions for a bucket. +// accessControlPolicy contains the elements providing ACL permissions +// for a bucket. type accessControlPolicy struct { // accessControlList container for ACL information. AccessControlList struct { diff --git a/Godeps/_workspace/src/github.com/minio/minio-go/api-stat.go b/Godeps/_workspace/src/github.com/minio/minio-go/api-stat.go index 8a29bccd5..826782033 100644 --- a/Godeps/_workspace/src/github.com/minio/minio-go/api-stat.go +++ b/Godeps/_workspace/src/github.com/minio/minio-go/api-stat.go @@ -25,15 +25,18 @@ import ( // BucketExists verify if bucket exists and you have permission to access it. func (c Client) BucketExists(bucketName string) error { + // Input validation. if err := isValidBucketName(bucketName); err != nil { return err } + // Instantiate a new request. req, err := c.newRequest("HEAD", requestMetadata{ bucketName: bucketName, }) if err != nil { return err } + // Initiate the request. resp, err := c.do(req) defer closeResponse(resp) if err != nil { @@ -48,12 +51,13 @@ func (c Client) BucketExists(bucketName string) error { } // StatObject verifies if object exists and you have permission to access. -func (c Client) StatObject(bucketName, objectName string) (ObjectStat, error) { +func (c Client) StatObject(bucketName, objectName string) (ObjectInfo, error) { + // Input validation. if err := isValidBucketName(bucketName); err != nil { - return ObjectStat{}, err + return ObjectInfo{}, err } if err := isValidObjectName(objectName); err != nil { - return ObjectStat{}, err + return ObjectInfo{}, err } // Instantiate a new request. req, err := c.newRequest("HEAD", requestMetadata{ @@ -61,16 +65,17 @@ func (c Client) StatObject(bucketName, objectName string) (ObjectStat, error) { objectName: objectName, }) if err != nil { - return ObjectStat{}, err + return ObjectInfo{}, err } + // Initiate the request. resp, err := c.do(req) defer closeResponse(resp) if err != nil { - return ObjectStat{}, err + return ObjectInfo{}, err } if resp != nil { if resp.StatusCode != http.StatusOK { - return ObjectStat{}, HTTPRespToErrorResponse(resp, bucketName, objectName) + return ObjectInfo{}, HTTPRespToErrorResponse(resp, bucketName, objectName) } } @@ -81,7 +86,7 @@ func (c Client) StatObject(bucketName, objectName string) (ObjectStat, error) { // Parse content length. size, err := strconv.ParseInt(resp.Header.Get("Content-Length"), 10, 64) if err != nil { - return ObjectStat{}, ErrorResponse{ + return ObjectInfo{}, ErrorResponse{ Code: "InternalError", Message: "Content-Length is invalid. " + reportIssue, BucketName: bucketName, @@ -91,9 +96,10 @@ func (c Client) StatObject(bucketName, objectName string) (ObjectStat, error) { AmzBucketRegion: resp.Header.Get("x-amz-bucket-region"), } } + // Parse Last-Modified has http time format. date, err := time.Parse(http.TimeFormat, resp.Header.Get("Last-Modified")) if err != nil { - return ObjectStat{}, ErrorResponse{ + return ObjectInfo{}, ErrorResponse{ Code: "InternalError", Message: "Last-Modified time format is invalid. " + reportIssue, BucketName: bucketName, @@ -103,12 +109,13 @@ func (c Client) StatObject(bucketName, objectName string) (ObjectStat, error) { AmzBucketRegion: resp.Header.Get("x-amz-bucket-region"), } } + // Fetch content type if any present. contentType := strings.TrimSpace(resp.Header.Get("Content-Type")) if contentType == "" { contentType = "application/octet-stream" } // Save object metadata info. - var objectStat ObjectStat + var objectStat ObjectInfo objectStat.ETag = md5sum objectStat.Key = objectName objectStat.Size = size diff --git a/Godeps/_workspace/src/github.com/minio/minio-go/api.go b/Godeps/_workspace/src/github.com/minio/minio-go/api.go index f74bf2036..9b7f3c077 100644 --- a/Godeps/_workspace/src/github.com/minio/minio-go/api.go +++ b/Godeps/_workspace/src/github.com/minio/minio-go/api.go @@ -25,6 +25,7 @@ import ( "net/http/httputil" "net/url" "os" + "regexp" "runtime" "strings" "time" @@ -33,10 +34,15 @@ import ( // Client implements Amazon S3 compatible methods. type Client struct { /// Standard options. - accessKeyID string // AccessKeyID required for authorized requests. - secretAccessKey string // SecretAccessKey required for authorized requests. - signature SignatureType // Choose a signature type if necessary. - anonymous bool // Set to 'true' if Client has no access and secret keys. + + // AccessKeyID required for authorized requests. + accessKeyID string + // SecretAccessKey required for authorized requests. + secretAccessKey string + // Choose a signature type if necessary. + signature SignatureType + // Set to 'true' if Client has no access and secret keys. + anonymous bool // User supplied. appInfo struct { @@ -69,7 +75,8 @@ const ( libraryUserAgent = libraryUserAgentPrefix + libraryName + "/" + libraryVersion ) -// NewV2 - instantiate minio client with Amazon S3 signature version '2' compatiblity. +// NewV2 - instantiate minio client with Amazon S3 signature version +// '2' compatiblity. func NewV2(endpoint string, accessKeyID, secretAccessKey string, insecure bool) (CloudStorageClient, error) { clnt, err := privateNew(endpoint, accessKeyID, secretAccessKey, insecure) if err != nil { @@ -80,7 +87,8 @@ func NewV2(endpoint string, accessKeyID, secretAccessKey string, insecure bool) return clnt, nil } -// NewV4 - instantiate minio client with Amazon S3 signature version '4' compatibility. +// NewV4 - instantiate minio client with Amazon S3 signature version +// '4' compatibility. func NewV4(endpoint string, accessKeyID, secretAccessKey string, insecure bool) (CloudStorageClient, error) { clnt, err := privateNew(endpoint, accessKeyID, secretAccessKey, insecure) if err != nil { @@ -91,13 +99,15 @@ func NewV4(endpoint string, accessKeyID, secretAccessKey string, insecure bool) return clnt, nil } -// New - instantiate minio client Client, adds automatic verification of signature. +// New - instantiate minio client Client, adds automatic verification +// of signature. func New(endpoint string, accessKeyID, secretAccessKey string, insecure bool) (CloudStorageClient, error) { clnt, err := privateNew(endpoint, accessKeyID, secretAccessKey, insecure) if err != nil { return nil, err } - // Google cloud storage should be set to signature V2, force it if not. + // Google cloud storage should be set to signature V2, force it if + // not. if isGoogleEndpoint(clnt.endpointURL) { clnt.signature = SignatureV2 } @@ -136,7 +146,8 @@ func privateNew(endpoint, accessKeyID, secretAccessKey string, insecure bool) (* // SetAppInfo - add application details to user agent. func (c *Client) SetAppInfo(appName string, appVersion string) { - // if app name and version is not set, we do not a new user agent. + // if app name and version is not set, we do not a new user + // agent. if appName != "" && appVersion != "" { c.appInfo = struct { appName string @@ -149,12 +160,13 @@ func (c *Client) SetAppInfo(appName string, appVersion string) { // SetCustomTransport - set new custom transport. func (c *Client) SetCustomTransport(customHTTPTransport http.RoundTripper) { - // Set this to override default transport ``http.DefaultTransport``. + // Set this to override default transport + // ``http.DefaultTransport``. // - // This transport is usually needed for debugging OR to add your own - // custom TLS certificates on the client transport, for custom CA's and - // certs which are not part of standard certificate authority follow this - // example :- + // This transport is usually needed for debugging OR to add your + // own custom TLS certificates on the client transport, for custom + // CA's and certs which are not part of standard certificate + // authority follow this example :- // // tr := &http.Transport{ // TLSClientConfig: &tls.Config{RootCAs: pool}, @@ -187,7 +199,8 @@ func (c *Client) TraceOff() { c.isTraceEnabled = false } -// requestMetadata - is container for all the values to make a request. +// requestMetadata - is container for all the values to make a +// request. type requestMetadata struct { // If set newRequest presigns the URL. presignURL bool @@ -202,10 +215,41 @@ type requestMetadata struct { // Generated by our internal code. contentBody io.ReadCloser contentLength int64 - contentSha256Bytes []byte + contentSHA256Bytes []byte contentMD5Bytes []byte } +// Filter out signature value from Authorization header. +func (c Client) filterSignature(req *http.Request) { + // For anonymous requests return here. + if c.anonymous { + return + } + // Handle if Signature V2. + if c.signature.isV2() { + // Set a temporary redacted auth + req.Header.Set("Authorization", "AWS **REDACTED**:**REDACTED**") + return + } + + /// Signature V4 authorization header. + + // Save the original auth. + origAuth := req.Header.Get("Authorization") + // Strip out accessKeyID from: + // Credential=////aws4_request + regCred := regexp.MustCompile("Credential=([A-Z0-9]+)/") + newAuth := regCred.ReplaceAllString(origAuth, "Credential=**REDACTED**/") + + // Strip out 256-bit signature from: Signature=<256-bit signature> + regSign := regexp.MustCompile("Signature=([[0-9a-f]+)") + newAuth = regSign.ReplaceAllString(newAuth, "Signature=**REDACTED**") + + // Set a temporary redacted auth + req.Header.Set("Authorization", newAuth) + return +} + // dumpHTTP - dump HTTP request and response. func (c Client) dumpHTTP(req *http.Request, resp *http.Response) error { // Starts http dump. @@ -214,6 +258,9 @@ func (c Client) dumpHTTP(req *http.Request, resp *http.Response) error { return err } + // Filter out Signature field from Authorization header. + c.filterSignature(req) + // Only display request header. reqTrace, err := httputil.DumpRequestOut(req, false) if err != nil { @@ -227,11 +274,22 @@ func (c Client) dumpHTTP(req *http.Request, resp *http.Response) error { } // Only display response header. - respTrace, err := httputil.DumpResponse(resp, false) - if err != nil { - return err - } + var respTrace []byte + // For errors we make sure to dump response body as well. + if resp.StatusCode != http.StatusOK && + resp.StatusCode != http.StatusPartialContent && + resp.StatusCode != http.StatusNoContent { + respTrace, err = httputil.DumpResponse(resp, true) + if err != nil { + return err + } + } else { + respTrace, err = httputil.DumpResponse(resp, false) + if err != nil { + return err + } + } // Write response to trace output. _, err = fmt.Fprint(c.traceOutput, strings.TrimSuffix(string(respTrace), "\r\n")) if err != nil { @@ -328,11 +386,12 @@ func (c Client) newRequest(method string, metadata requestMetadata) (*http.Reque // Set sha256 sum only for non anonymous credentials. if !c.anonymous { - // set sha256 sum for signature calculation only with signature version '4'. + // set sha256 sum for signature calculation only with + // signature version '4'. if c.signature.isV4() { req.Header.Set("X-Amz-Content-Sha256", hex.EncodeToString(sum256([]byte{}))) - if metadata.contentSha256Bytes != nil { - req.Header.Set("X-Amz-Content-Sha256", hex.EncodeToString(metadata.contentSha256Bytes)) + if metadata.contentSHA256Bytes != nil { + req.Header.Set("X-Amz-Content-Sha256", hex.EncodeToString(metadata.contentSHA256Bytes)) } } } @@ -356,6 +415,7 @@ func (c Client) newRequest(method string, metadata requestMetadata) (*http.Reque return req, nil } +// set User agent. func (c Client) setUserAgent(req *http.Request) { req.Header.Set("User-Agent", libraryUserAgent) if c.appInfo.appName != "" && c.appInfo.appVersion != "" { @@ -363,12 +423,15 @@ func (c Client) setUserAgent(req *http.Request) { } } +// makeTargetURL make a new target url. func (c Client) makeTargetURL(bucketName, objectName string, queryValues url.Values) (*url.URL, error) { urlStr := c.endpointURL.Scheme + "://" + c.endpointURL.Host + "/" - // Make URL only if bucketName is available, otherwise use the endpoint URL. + // Make URL only if bucketName is available, otherwise use the + // endpoint URL. if bucketName != "" { // If endpoint supports virtual host style use that always. - // Currently only S3 and Google Cloud Storage would support this. + // Currently only S3 and Google Cloud Storage would support + // this. if isVirtualHostSupported(c.endpointURL) { urlStr = c.endpointURL.Scheme + "://" + bucketName + "." + c.endpointURL.Host + "/" if objectName != "" { @@ -403,21 +466,17 @@ type CloudStorageClient interface { SetBucketACL(bucketName string, cannedACL BucketACL) error GetBucketACL(bucketName string) (BucketACL, error) - ListBuckets() ([]BucketStat, error) - ListObjects(bucket, prefix string, recursive bool, doneCh <-chan struct{}) <-chan ObjectStat - ListIncompleteUploads(bucket, prefix string, recursive bool, doneCh <-chan struct{}) <-chan ObjectMultipartStat + ListBuckets() ([]BucketInfo, error) + ListObjects(bucket, prefix string, recursive bool, doneCh <-chan struct{}) <-chan ObjectInfo + ListIncompleteUploads(bucket, prefix string, recursive bool, doneCh <-chan struct{}) <-chan ObjectMultipartInfo // Object Read/Write/Stat operations. - GetObject(bucketName, objectName string) (reader io.ReadCloser, stat ObjectStat, err error) - PutObject(bucketName, objectName string, data io.Reader, size int64, contentType string) (n int64, err error) - StatObject(bucketName, objectName string) (ObjectStat, error) + GetObject(bucketName, objectName string) (reader *Object, err error) + PutObject(bucketName, objectName string, reader io.Reader, contentType string) (n int64, err error) + StatObject(bucketName, objectName string) (ObjectInfo, error) RemoveObject(bucketName, objectName string) error RemoveIncompleteUpload(bucketName, objectName string) error - // Object Read/Write for sparse upload. - GetObjectPartial(bucketName, objectName string) (reader ReadAtCloser, stat ObjectStat, err error) - PutObjectPartial(bucketName, objectName string, data ReadAtCloser, size int64, contentType string) (n int64, err error) - // File to Object API. FPutObject(bucketName, objectName, filePath, contentType string) (n int64, err error) FGetObject(bucketName, objectName, filePath string) error diff --git a/Godeps/_workspace/src/github.com/minio/minio-go/api_functional_v2_test.go b/Godeps/_workspace/src/github.com/minio/minio-go/api_functional_v2_test.go new file mode 100644 index 000000000..51ba285c3 --- /dev/null +++ b/Godeps/_workspace/src/github.com/minio/minio-go/api_functional_v2_test.go @@ -0,0 +1,751 @@ +/* + * Minio Go Library for Amazon S3 Compatible Cloud Storage (C) 2015 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package minio_test + +import ( + "bytes" + crand "crypto/rand" + "errors" + "io" + "io/ioutil" + "math/rand" + "net/http" + "os" + "testing" + "time" + + "github.com/minio/minio-go" +) + +// Tests removing partially uploaded objects. +func TestRemovePartiallyUploadedV2(t *testing.T) { + if testing.Short() { + t.Skip("skipping function tests for short runs") + } + + // Seed random based on current time. + rand.Seed(time.Now().Unix()) + + // Connect and make sure bucket exists. + c, err := minio.NewV2( + "s3.amazonaws.com", + os.Getenv("ACCESS_KEY"), + os.Getenv("SECRET_KEY"), + false, + ) + if err != nil { + t.Fatal("Error:", err) + } + + // Set user agent. + c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + + // Enable tracing, write to stdout. + // c.TraceOn(os.Stderr) + + // Generate a new random bucket name. + bucketName := randString(60, rand.NewSource(time.Now().UnixNano())) + + // make a new bucket. + err = c.MakeBucket(bucketName, "private", "us-east-1") + if err != nil { + t.Fatal("Error:", err, bucketName) + } + + reader, writer := io.Pipe() + go func() { + i := 0 + for i < 25 { + _, err = io.CopyN(writer, crand.Reader, 128*1024) + if err != nil { + t.Fatal("Error:", err, bucketName) + } + i++ + } + writer.CloseWithError(errors.New("Proactively closed to be verified later.")) + }() + + objectName := bucketName + "-resumable" + _, err = c.PutObject(bucketName, objectName, reader, "application/octet-stream") + if err == nil { + t.Fatal("Error: PutObject should fail.") + } + if err.Error() != "Proactively closed to be verified later." { + t.Fatal("Error:", err) + } + err = c.RemoveIncompleteUpload(bucketName, objectName) + if err != nil { + t.Fatal("Error:", err) + } + err = c.RemoveBucket(bucketName) + if err != nil { + t.Fatal("Error:", err) + } +} + +// Tests resumable file based put object multipart upload. +func TestResumableFPutObjectV2(t *testing.T) { + if testing.Short() { + t.Skip("skipping functional tests for the short runs") + } + + // Seed random based on current time. + rand.Seed(time.Now().Unix()) + + // Connect and make sure bucket exists. + c, err := minio.New( + "s3.amazonaws.com", + os.Getenv("ACCESS_KEY"), + os.Getenv("SECRET_KEY"), + false, + ) + if err != nil { + t.Fatal("Error:", err) + } + + // Set user agent. + c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + + // Enable tracing, write to stdout. + // c.TraceOn(os.Stderr) + + // Generate a new random bucket name. + bucketName := randString(60, rand.NewSource(time.Now().UnixNano())) + + // make a new bucket. + err = c.MakeBucket(bucketName, "private", "us-east-1") + if err != nil { + t.Fatal("Error:", err, bucketName) + } + + file, err := ioutil.TempFile(os.TempDir(), "resumable") + if err != nil { + t.Fatal("Error:", err) + } + + n, err := io.CopyN(file, crand.Reader, 11*1024*1024) + if err != nil { + t.Fatal("Error:", err) + } + if n != int64(11*1024*1024) { + t.Fatalf("Error: number of bytes does not match, want %v, got %v\n", 11*1024*1024, n) + } + + objectName := bucketName + "-resumable" + + n, err = c.FPutObject(bucketName, objectName, file.Name(), "application/octet-stream") + if err != nil { + t.Fatal("Error:", err) + } + if n != int64(11*1024*1024) { + t.Fatalf("Error: number of bytes does not match, want %v, got %v\n", 11*1024*1024, n) + } + + // Close the file pro-actively for windows. + file.Close() + + err = c.RemoveObject(bucketName, objectName) + if err != nil { + t.Fatal("Error: ", err) + } + + err = c.RemoveBucket(bucketName) + if err != nil { + t.Fatal("Error:", err) + } + + err = os.Remove(file.Name()) + if err != nil { + t.Fatal("Error:", err) + } +} + +// Tests resumable put object multipart upload. +func TestResumablePutObjectV2(t *testing.T) { + if testing.Short() { + t.Skip("skipping functional tests for the short runs") + } + + // Seed random based on current time. + rand.Seed(time.Now().Unix()) + + // Connect and make sure bucket exists. + c, err := minio.New( + "s3.amazonaws.com", + os.Getenv("ACCESS_KEY"), + os.Getenv("SECRET_KEY"), + false, + ) + if err != nil { + t.Fatal("Error:", err) + } + + // Enable tracing, write to stderr. + // c.TraceOn(os.Stderr) + + // Set user agent. + c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + + // Generate a new random bucket name. + bucketName := randString(60, rand.NewSource(time.Now().UnixNano())) + + // make a new bucket. + err = c.MakeBucket(bucketName, "private", "us-east-1") + if err != nil { + t.Fatal("Error:", err, bucketName) + } + + // generate 11MB + buf := make([]byte, 11*1024*1024) + + _, err = io.ReadFull(crand.Reader, buf) + if err != nil { + t.Fatal("Error:", err) + } + + objectName := bucketName + "-resumable" + reader := bytes.NewReader(buf) + n, err := c.PutObject(bucketName, objectName, reader, "application/octet-stream") + if err != nil { + t.Fatal("Error:", err, bucketName, objectName) + } + if n != int64(len(buf)) { + t.Fatalf("Error: number of bytes does not match, want %v, got %v\n", len(buf), n) + } + + err = c.RemoveObject(bucketName, objectName) + if err != nil { + t.Fatal("Error: ", err) + } + + err = c.RemoveBucket(bucketName) + if err != nil { + t.Fatal("Error:", err) + } +} + +// Tests get object ReaderSeeker interface methods. +func TestGetObjectReadSeekFunctionalV2(t *testing.T) { + if testing.Short() { + t.Skip("skipping functional tests for short runs") + } + + // Seed random based on current time. + rand.Seed(time.Now().Unix()) + + // Connect and make sure bucket exists. + c, err := minio.New( + "s3.amazonaws.com", + os.Getenv("ACCESS_KEY"), + os.Getenv("SECRET_KEY"), + false, + ) + if err != nil { + t.Fatal("Error:", err) + } + + // Enable tracing, write to stderr. + // c.TraceOn(os.Stderr) + + // Set user agent. + c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + + // Generate a new random bucket name. + bucketName := randString(60, rand.NewSource(time.Now().UnixNano())) + + // Make a new bucket. + err = c.MakeBucket(bucketName, "private", "us-east-1") + if err != nil { + t.Fatal("Error:", err, bucketName) + } + + // Generate data more than 32K + buf := make([]byte, rand.Intn(1<<20)+32*1024) + + _, err = io.ReadFull(crand.Reader, buf) + if err != nil { + t.Fatal("Error:", err) + } + + // Save the data + objectName := randString(60, rand.NewSource(time.Now().UnixNano())) + n, err := c.PutObject(bucketName, objectName, bytes.NewReader(buf), "binary/octet-stream") + if err != nil { + t.Fatal("Error:", err, bucketName, objectName) + } + + if n != int64(len(buf)) { + t.Fatalf("Error: number of bytes does not match, want %v, got %v\n", len(buf), n) + } + + // Read the data back + r, err := c.GetObject(bucketName, objectName) + if err != nil { + t.Fatal("Error:", err, bucketName, objectName) + } + + st, err := r.Stat() + if err != nil { + t.Fatal("Error:", err, bucketName, objectName) + } + if st.Size != int64(len(buf)) { + t.Fatalf("Error: number of bytes in stat does not match, want %v, got %v\n", + len(buf), st.Size) + } + + offset := int64(2048) + n, err = r.Seek(offset, 0) + if err != nil { + t.Fatal("Error:", err, offset) + } + if n != offset { + t.Fatalf("Error: number of bytes seeked does not match, want %v, got %v\n", + offset, n) + } + n, err = r.Seek(0, 1) + if err != nil { + t.Fatal("Error:", err) + } + if n != offset { + t.Fatalf("Error: number of current seek does not match, want %v, got %v\n", + offset, n) + } + _, err = r.Seek(offset, 2) + if err == nil { + t.Fatal("Error: seek on positive offset for whence '2' should error out") + } + n, err = r.Seek(-offset, 2) + if err != nil { + t.Fatal("Error:", err) + } + if n != 0 { + t.Fatalf("Error: number of bytes seeked back does not match, want 0, got %v\n", n) + } + var buffer bytes.Buffer + if _, err = io.CopyN(&buffer, r, st.Size); err != nil { + t.Fatal("Error:", err) + } + if !bytes.Equal(buf, buffer.Bytes()) { + t.Fatal("Error: Incorrect read bytes v/s original buffer.") + } + err = c.RemoveObject(bucketName, objectName) + if err != nil { + t.Fatal("Error: ", err) + } + err = c.RemoveBucket(bucketName) + if err != nil { + t.Fatal("Error:", err) + } +} + +// Tests get object ReaderAt interface methods. +func TestGetObjectReadAtFunctionalV2(t *testing.T) { + if testing.Short() { + t.Skip("skipping functional tests for the short runs") + } + + // Seed random based on current time. + rand.Seed(time.Now().Unix()) + + // Connect and make sure bucket exists. + c, err := minio.New( + "s3.amazonaws.com", + os.Getenv("ACCESS_KEY"), + os.Getenv("SECRET_KEY"), + false, + ) + if err != nil { + t.Fatal("Error:", err) + } + + // Enable tracing, write to stderr. + // c.TraceOn(os.Stderr) + + // Set user agent. + c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + + // Generate a new random bucket name. + bucketName := randString(60, rand.NewSource(time.Now().UnixNano())) + + // Make a new bucket. + err = c.MakeBucket(bucketName, "private", "us-east-1") + if err != nil { + t.Fatal("Error:", err, bucketName) + } + + // Generate data more than 32K + buf := make([]byte, rand.Intn(1<<20)+32*1024) + + _, err = io.ReadFull(crand.Reader, buf) + if err != nil { + t.Fatal("Error:", err) + } + + // Save the data + objectName := randString(60, rand.NewSource(time.Now().UnixNano())) + n, err := c.PutObject(bucketName, objectName, bytes.NewReader(buf), "binary/octet-stream") + if err != nil { + t.Fatal("Error:", err, bucketName, objectName) + } + + if n != int64(len(buf)) { + t.Fatalf("Error: number of bytes does not match, want %v, got %v\n", len(buf), n) + } + + // Read the data back + r, err := c.GetObject(bucketName, objectName) + if err != nil { + t.Fatal("Error:", err, bucketName, objectName) + } + + st, err := r.Stat() + if err != nil { + t.Fatal("Error:", err, bucketName, objectName) + } + if st.Size != int64(len(buf)) { + t.Fatalf("Error: number of bytes in stat does not match, want %v, got %v\n", + len(buf), st.Size) + } + + offset := int64(2048) + + // Read directly + buf2 := make([]byte, 512) + buf3 := make([]byte, 512) + buf4 := make([]byte, 512) + + m, err := r.ReadAt(buf2, offset) + if err != nil { + t.Fatal("Error:", err, st.Size, len(buf2), offset) + } + if m != len(buf2) { + t.Fatalf("Error: ReadAt read shorter bytes before reaching EOF, want %v, got %v\n", m, len(buf2)) + } + if !bytes.Equal(buf2, buf[offset:offset+512]) { + t.Fatal("Error: Incorrect read between two ReadAt from same offset.") + } + offset += 512 + m, err = r.ReadAt(buf3, offset) + if err != nil { + t.Fatal("Error:", err, st.Size, len(buf3), offset) + } + if m != len(buf3) { + t.Fatalf("Error: ReadAt read shorter bytes before reaching EOF, want %v, got %v\n", m, len(buf3)) + } + if !bytes.Equal(buf3, buf[offset:offset+512]) { + t.Fatal("Error: Incorrect read between two ReadAt from same offset.") + } + offset += 512 + m, err = r.ReadAt(buf4, offset) + if err != nil { + t.Fatal("Error:", err, st.Size, len(buf4), offset) + } + if m != len(buf4) { + t.Fatalf("Error: ReadAt read shorter bytes before reaching EOF, want %v, got %v\n", m, len(buf4)) + } + if !bytes.Equal(buf4, buf[offset:offset+512]) { + t.Fatal("Error: Incorrect read between two ReadAt from same offset.") + } + + buf5 := make([]byte, n) + // Read the whole object. + m, err = r.ReadAt(buf5, 0) + if err != nil { + if err != io.EOF { + t.Fatal("Error:", err, len(buf5)) + } + } + if m != len(buf5) { + t.Fatalf("Error: ReadAt read shorter bytes before reaching EOF, want %v, got %v\n", m, len(buf5)) + } + if !bytes.Equal(buf, buf5) { + t.Fatal("Error: Incorrect data read in GetObject, than what was previously upoaded.") + } + + buf6 := make([]byte, n+1) + // Read the whole object and beyond. + _, err = r.ReadAt(buf6, 0) + if err != nil { + if err != io.EOF { + t.Fatal("Error:", err, len(buf6)) + } + } + err = c.RemoveObject(bucketName, objectName) + if err != nil { + t.Fatal("Error: ", err) + } + err = c.RemoveBucket(bucketName) + if err != nil { + t.Fatal("Error:", err) + } +} + +// Tests comprehensive list of all methods. +func TestFunctionalV2(t *testing.T) { + if testing.Short() { + t.Skip("skipping functional tests for the short runs") + } + + // Seed random based on current time. + rand.Seed(time.Now().Unix()) + + c, err := minio.New( + "s3.amazonaws.com", + os.Getenv("ACCESS_KEY"), + os.Getenv("SECRET_KEY"), + false, + ) + if err != nil { + t.Fatal("Error:", err) + } + + // Enable to debug + // c.TraceOn(os.Stderr) + + // Set user agent. + c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + + // Generate a new random bucket name. + bucketName := randString(60, rand.NewSource(time.Now().UnixNano())) + + // Make a new bucket. + err = c.MakeBucket(bucketName, "private", "us-east-1") + if err != nil { + t.Fatal("Error:", err, bucketName) + } + + // Generate a random file name. + fileName := randString(60, rand.NewSource(time.Now().UnixNano())) + file, err := os.Create(fileName) + if err != nil { + t.Fatal("Error:", err) + } + var totalSize int64 + for i := 0; i < 3; i++ { + buf := make([]byte, rand.Intn(1<<19)) + n, err := file.Write(buf) + if err != nil { + t.Fatal("Error:", err) + } + totalSize += int64(n) + } + file.Close() + + // Verify if bucket exits and you have access. + err = c.BucketExists(bucketName) + if err != nil { + t.Fatal("Error:", err, bucketName) + } + + // Make the bucket 'public read/write'. + err = c.SetBucketACL(bucketName, "public-read-write") + if err != nil { + t.Fatal("Error:", err) + } + + // Get the previously set acl. + acl, err := c.GetBucketACL(bucketName) + if err != nil { + t.Fatal("Error:", err) + } + + // ACL must be 'public read/write'. + if acl != minio.BucketACL("public-read-write") { + t.Fatal("Error:", acl) + } + + // List all buckets. + buckets, err := c.ListBuckets() + if len(buckets) == 0 { + t.Fatal("Error: list buckets cannot be empty", buckets) + } + if err != nil { + t.Fatal("Error:", err) + } + + // Verify if previously created bucket is listed in list buckets. + bucketFound := false + for _, bucket := range buckets { + if bucket.Name == bucketName { + bucketFound = true + } + } + + // If bucket not found error out. + if !bucketFound { + t.Fatal("Error: bucket ", bucketName, "not found") + } + + objectName := bucketName + "unique" + + // Generate data + buf := make([]byte, rand.Intn(1<<19)) + _, err = io.ReadFull(crand.Reader, buf) + if err != nil { + t.Fatal("Error: ", err) + } + + n, err := c.PutObject(bucketName, objectName, bytes.NewReader(buf), "") + if err != nil { + t.Fatal("Error: ", err) + } + if n != int64(len(buf)) { + t.Fatal("Error: bad length ", n, len(buf)) + } + + n, err = c.PutObject(bucketName, objectName+"-nolength", bytes.NewReader(buf), "binary/octet-stream") + if err != nil { + t.Fatal("Error:", err, bucketName, objectName+"-nolength") + } + + if n != int64(len(buf)) { + t.Fatalf("Error: number of bytes does not match, want %v, got %v\n", len(buf), n) + } + + // Instantiate a done channel to close all listing. + doneCh := make(chan struct{}) + defer close(doneCh) + + objFound := false + isRecursive := true // Recursive is true. + for obj := range c.ListObjects(bucketName, objectName, isRecursive, doneCh) { + if obj.Key == objectName { + objFound = true + break + } + } + if !objFound { + t.Fatal("Error: object " + objectName + " not found.") + } + + incompObjNotFound := true + for objIncompl := range c.ListIncompleteUploads(bucketName, objectName, isRecursive, doneCh) { + if objIncompl.Key != "" { + incompObjNotFound = false + break + } + } + if !incompObjNotFound { + t.Fatal("Error: unexpected dangling incomplete upload found.") + } + + newReader, err := c.GetObject(bucketName, objectName) + if err != nil { + t.Fatal("Error: ", err) + } + + newReadBytes, err := ioutil.ReadAll(newReader) + if err != nil { + t.Fatal("Error: ", err) + } + + if !bytes.Equal(newReadBytes, buf) { + t.Fatal("Error: bytes mismatch.") + } + + err = c.FGetObject(bucketName, objectName, fileName+"-f") + if err != nil { + t.Fatal("Error: ", err) + } + + presignedGetURL, err := c.PresignedGetObject(bucketName, objectName, 3600*time.Second) + if err != nil { + t.Fatal("Error: ", err) + } + + resp, err := http.Get(presignedGetURL) + if err != nil { + t.Fatal("Error: ", err) + } + if resp.StatusCode != http.StatusOK { + t.Fatal("Error: ", resp.Status) + } + newPresignedBytes, err := ioutil.ReadAll(resp.Body) + if err != nil { + t.Fatal("Error: ", err) + } + if !bytes.Equal(newPresignedBytes, buf) { + t.Fatal("Error: bytes mismatch.") + } + + presignedPutURL, err := c.PresignedPutObject(bucketName, objectName+"-presigned", 3600*time.Second) + if err != nil { + t.Fatal("Error: ", err) + } + buf = make([]byte, rand.Intn(1<<20)) + _, err = io.ReadFull(crand.Reader, buf) + if err != nil { + t.Fatal("Error: ", err) + } + req, err := http.NewRequest("PUT", presignedPutURL, bytes.NewReader(buf)) + if err != nil { + t.Fatal("Error: ", err) + } + httpClient := &http.Client{} + resp, err = httpClient.Do(req) + if err != nil { + t.Fatal("Error: ", err) + } + + newReader, err = c.GetObject(bucketName, objectName+"-presigned") + if err != nil { + t.Fatal("Error: ", err) + } + + newReadBytes, err = ioutil.ReadAll(newReader) + if err != nil { + t.Fatal("Error: ", err) + } + + if !bytes.Equal(newReadBytes, buf) { + t.Fatal("Error: bytes mismatch.") + } + + err = c.RemoveObject(bucketName, objectName) + if err != nil { + t.Fatal("Error: ", err) + } + err = c.RemoveObject(bucketName, objectName+"-f") + if err != nil { + t.Fatal("Error: ", err) + } + err = c.RemoveObject(bucketName, objectName+"-nolength") + if err != nil { + t.Fatal("Error: ", err) + } + err = c.RemoveObject(bucketName, objectName+"-presigned") + if err != nil { + t.Fatal("Error: ", err) + } + err = c.RemoveBucket(bucketName) + if err != nil { + t.Fatal("Error:", err) + } + err = c.RemoveBucket(bucketName) + if err == nil { + t.Fatal("Error:") + } + if err.Error() != "The specified bucket does not exist" { + t.Fatal("Error: ", err) + } + if err = os.Remove(fileName); err != nil { + t.Fatal("Error: ", err) + } + if err = os.Remove(fileName + "-f"); err != nil { + t.Fatal("Error: ", err) + } +} diff --git a/Godeps/_workspace/src/github.com/minio/minio-go/api_functional_test.go b/Godeps/_workspace/src/github.com/minio/minio-go/api_functional_v4_test.go similarity index 62% rename from Godeps/_workspace/src/github.com/minio/minio-go/api_functional_test.go rename to Godeps/_workspace/src/github.com/minio/minio-go/api_functional_v4_test.go index f7bd81097..d452d8484 100644 --- a/Godeps/_workspace/src/github.com/minio/minio-go/api_functional_test.go +++ b/Godeps/_workspace/src/github.com/minio/minio-go/api_functional_v4_test.go @@ -19,6 +19,7 @@ package minio_test import ( "bytes" crand "crypto/rand" + "errors" "io" "io/ioutil" "math/rand" @@ -54,9 +55,10 @@ func randString(n int, src rand.Source) string { return string(b[0:30]) } -func TestResumableFPutObject(t *testing.T) { +// Tests removing partially uploaded objects. +func TestRemovePartiallyUploaded(t *testing.T) { if testing.Short() { - t.Skip("skipping resumable tests with short runs") + t.Skip("skipping function tests for short runs") } // Seed random based on current time. @@ -64,9 +66,9 @@ func TestResumableFPutObject(t *testing.T) { // Connect and make sure bucket exists. c, err := minio.New( - "play.minio.io:9002", - "Q3AM3UQ867SPQQA43P2F", - "zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG", + "s3.amazonaws.com", + os.Getenv("ACCESS_KEY"), + os.Getenv("SECRET_KEY"), false, ) if err != nil { @@ -77,12 +79,78 @@ func TestResumableFPutObject(t *testing.T) { c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") // Enable tracing, write to stdout. - // c.TraceOn(nil) + // c.TraceOn(os.Stderr) // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano())) - // make a new bucket. + // Make a new bucket. + err = c.MakeBucket(bucketName, "private", "us-east-1") + if err != nil { + t.Fatal("Error:", err, bucketName) + } + + reader, writer := io.Pipe() + go func() { + i := 0 + for i < 25 { + _, err = io.CopyN(writer, crand.Reader, 128*1024) + if err != nil { + t.Fatal("Error:", err, bucketName) + } + i++ + } + writer.CloseWithError(errors.New("Proactively closed to be verified later.")) + }() + + objectName := bucketName + "-resumable" + _, err = c.PutObject(bucketName, objectName, reader, "application/octet-stream") + if err == nil { + t.Fatal("Error: PutObject should fail.") + } + if err.Error() != "Proactively closed to be verified later." { + t.Fatal("Error:", err) + } + err = c.RemoveIncompleteUpload(bucketName, objectName) + if err != nil { + t.Fatal("Error:", err) + } + err = c.RemoveBucket(bucketName) + if err != nil { + t.Fatal("Error:", err) + } +} + +// Tests resumable file based put object multipart upload. +func TestResumableFPutObject(t *testing.T) { + if testing.Short() { + t.Skip("skipping functional tests for the short runs") + } + + // Seed random based on current time. + rand.Seed(time.Now().Unix()) + + // Connect and make sure bucket exists. + c, err := minio.New( + "s3.amazonaws.com", + os.Getenv("ACCESS_KEY"), + os.Getenv("SECRET_KEY"), + false, + ) + if err != nil { + t.Fatal("Error:", err) + } + + // Set user agent. + c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + + // Enable tracing, write to stdout. + // c.TraceOn(os.Stderr) + + // Generate a new random bucket name. + bucketName := randString(60, rand.NewSource(time.Now().UnixNano())) + + // Make a new bucket. err = c.MakeBucket(bucketName, "private", "us-east-1") if err != nil { t.Fatal("Error:", err, bucketName) @@ -93,7 +161,10 @@ func TestResumableFPutObject(t *testing.T) { t.Fatal("Error:", err) } - n, _ := io.CopyN(file, crand.Reader, 11*1024*1024) + n, err := io.CopyN(file, crand.Reader, 11*1024*1024) + if err != nil { + t.Fatal("Error:", err) + } if n != int64(11*1024*1024) { t.Fatalf("Error: number of bytes does not match, want %v, got %v\n", 11*1024*1024, n) } @@ -127,9 +198,10 @@ func TestResumableFPutObject(t *testing.T) { } } +// Tests resumable put object multipart upload. func TestResumablePutObject(t *testing.T) { if testing.Short() { - t.Skip("skipping resumable tests with short runs") + t.Skip("skipping functional tests for the short runs") } // Seed random based on current time. @@ -137,31 +209,31 @@ func TestResumablePutObject(t *testing.T) { // Connect and make sure bucket exists. c, err := minio.New( - "play.minio.io:9002", - "Q3AM3UQ867SPQQA43P2F", - "zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG", + "s3.amazonaws.com", + os.Getenv("ACCESS_KEY"), + os.Getenv("SECRET_KEY"), false, ) if err != nil { t.Fatal("Error:", err) } + // Enable tracing, write to stderr. + // c.TraceOn(os.Stderr) + // Set user agent. c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") - // Enable tracing, write to stdout. - // c.TraceOn(nil) - // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano())) - // make a new bucket. + // Make a new bucket. err = c.MakeBucket(bucketName, "private", "us-east-1") if err != nil { t.Fatal("Error:", err, bucketName) } - // generate 11MB + // Generate 11MB buf := make([]byte, 11*1024*1024) _, err = io.ReadFull(crand.Reader, buf) @@ -171,7 +243,7 @@ func TestResumablePutObject(t *testing.T) { objectName := bucketName + "-resumable" reader := bytes.NewReader(buf) - n, err := c.PutObject(bucketName, objectName, reader, int64(reader.Len()), "application/octet-stream") + n, err := c.PutObject(bucketName, objectName, reader, "application/octet-stream") if err != nil { t.Fatal("Error:", err, bucketName, objectName) } @@ -190,37 +262,42 @@ func TestResumablePutObject(t *testing.T) { } } -func TestGetObjectPartialFunctional(t *testing.T) { +// Tests get object ReaderSeeker interface methods. +func TestGetObjectReadSeekFunctional(t *testing.T) { + if testing.Short() { + t.Skip("skipping functional tests for short runs") + } + // Seed random based on current time. rand.Seed(time.Now().Unix()) // Connect and make sure bucket exists. c, err := minio.New( - "play.minio.io:9002", - "Q3AM3UQ867SPQQA43P2F", - "zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG", + "s3.amazonaws.com", + os.Getenv("ACCESS_KEY"), + os.Getenv("SECRET_KEY"), false, ) if err != nil { t.Fatal("Error:", err) } + // Enable tracing, write to stderr. + // c.TraceOn(os.Stderr) + // Set user agent. c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") - // Enable tracing, write to stdout. - // c.TraceOn(nil) - // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano())) - // make a new bucket. + // Make a new bucket. err = c.MakeBucket(bucketName, "private", "us-east-1") if err != nil { t.Fatal("Error:", err, bucketName) } - // generate data more than 32K + // Generate data more than 32K buf := make([]byte, rand.Intn(1<<20)+32*1024) _, err = io.ReadFull(crand.Reader, buf) @@ -228,9 +305,123 @@ func TestGetObjectPartialFunctional(t *testing.T) { t.Fatal("Error:", err) } - // save the data + // Save the data objectName := randString(60, rand.NewSource(time.Now().UnixNano())) - n, err := c.PutObject(bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), "binary/octet-stream") + n, err := c.PutObject(bucketName, objectName, bytes.NewReader(buf), "binary/octet-stream") + if err != nil { + t.Fatal("Error:", err, bucketName, objectName) + } + + if n != int64(len(buf)) { + t.Fatalf("Error: number of bytes does not match, want %v, got %v\n", len(buf), n) + } + + // Read the data back + r, err := c.GetObject(bucketName, objectName) + if err != nil { + t.Fatal("Error:", err, bucketName, objectName) + } + + st, err := r.Stat() + if err != nil { + t.Fatal("Error:", err, bucketName, objectName) + } + if st.Size != int64(len(buf)) { + t.Fatalf("Error: number of bytes in stat does not match, want %v, got %v\n", + len(buf), st.Size) + } + + offset := int64(2048) + n, err = r.Seek(offset, 0) + if err != nil { + t.Fatal("Error:", err, offset) + } + if n != offset { + t.Fatalf("Error: number of bytes seeked does not match, want %v, got %v\n", + offset, n) + } + n, err = r.Seek(0, 1) + if err != nil { + t.Fatal("Error:", err) + } + if n != offset { + t.Fatalf("Error: number of current seek does not match, want %v, got %v\n", + offset, n) + } + _, err = r.Seek(offset, 2) + if err == nil { + t.Fatal("Error: seek on positive offset for whence '2' should error out") + } + n, err = r.Seek(-offset, 2) + if err != nil { + t.Fatal("Error:", err) + } + if n != 0 { + t.Fatalf("Error: number of bytes seeked back does not match, want 0, got %v\n", n) + } + var buffer bytes.Buffer + if _, err = io.CopyN(&buffer, r, st.Size); err != nil { + t.Fatal("Error:", err) + } + if !bytes.Equal(buf, buffer.Bytes()) { + t.Fatal("Error: Incorrect read bytes v/s original buffer.") + } + err = c.RemoveObject(bucketName, objectName) + if err != nil { + t.Fatal("Error: ", err) + } + err = c.RemoveBucket(bucketName) + if err != nil { + t.Fatal("Error:", err) + } +} + +// Tests get object ReaderAt interface methods. +func TestGetObjectReadAtFunctional(t *testing.T) { + if testing.Short() { + t.Skip("skipping functional tests for the short runs") + } + + // Seed random based on current time. + rand.Seed(time.Now().Unix()) + + // Connect and make sure bucket exists. + c, err := minio.New( + "s3.amazonaws.com", + os.Getenv("ACCESS_KEY"), + os.Getenv("SECRET_KEY"), + false, + ) + if err != nil { + t.Fatal("Error:", err) + } + + // Enable tracing, write to stderr. + // c.TraceOn(os.Stderr) + + // Set user agent. + c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + + // Generate a new random bucket name. + bucketName := randString(60, rand.NewSource(time.Now().UnixNano())) + + // Make a new bucket. + err = c.MakeBucket(bucketName, "private", "us-east-1") + if err != nil { + t.Fatal("Error:", err, bucketName) + } + + // Generate data more than 32K + buf := make([]byte, rand.Intn(1<<20)+32*1024) + + _, err = io.ReadFull(crand.Reader, buf) + if err != nil { + t.Fatal("Error:", err) + } + + // Save the data + objectName := randString(60, rand.NewSource(time.Now().UnixNano())) + n, err := c.PutObject(bucketName, objectName, bytes.NewReader(buf), "binary/octet-stream") if err != nil { t.Fatal("Error:", err, bucketName, objectName) } @@ -240,11 +431,15 @@ func TestGetObjectPartialFunctional(t *testing.T) { } // read the data back - r, st, err := c.GetObjectPartial(bucketName, objectName) + r, err := c.GetObject(bucketName, objectName) if err != nil { t.Fatal("Error:", err, bucketName, objectName) } + st, err := r.Stat() + if err != nil { + t.Fatal("Error:", err, bucketName, objectName) + } if st.Size != int64(len(buf)) { t.Fatalf("Error: number of bytes in stat does not match, want %v, got %v\n", len(buf), st.Size) @@ -323,36 +518,41 @@ func TestGetObjectPartialFunctional(t *testing.T) { } } +// Tests comprehensive list of all methods. func TestFunctional(t *testing.T) { + if testing.Short() { + t.Skip("skipping functional tests for the short runs") + } + // Seed random based on current time. rand.Seed(time.Now().Unix()) c, err := minio.New( - "play.minio.io:9002", - "Q3AM3UQ867SPQQA43P2F", - "zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG", + "s3.amazonaws.com", + os.Getenv("ACCESS_KEY"), + os.Getenv("SECRET_KEY"), false, ) if err != nil { t.Fatal("Error:", err) } + // Enable to debug + // c.TraceOn(os.Stderr) + // Set user agent. c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") - // Enable tracing, write to stdout. - // c.TraceOn(nil) - // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano())) - // make a new bucket. + // Make a new bucket. err = c.MakeBucket(bucketName, "private", "us-east-1") if err != nil { t.Fatal("Error:", err, bucketName) } - // generate a random file name. + // Generate a random file name. fileName := randString(60, rand.NewSource(time.Now().UnixNano())) file, err := os.Create(fileName) if err != nil { @@ -369,31 +569,34 @@ func TestFunctional(t *testing.T) { } file.Close() - // verify if bucket exits and you have access. + // Verify if bucket exits and you have access. err = c.BucketExists(bucketName) if err != nil { t.Fatal("Error:", err, bucketName) } - // make the bucket 'public read/write'. + // Make the bucket 'public read/write'. err = c.SetBucketACL(bucketName, "public-read-write") if err != nil { t.Fatal("Error:", err) } - // get the previously set acl. + // Get the previously set acl. acl, err := c.GetBucketACL(bucketName) if err != nil { t.Fatal("Error:", err) } - // acl must be 'public read/write'. + // ACL must be 'public read/write'. if acl != minio.BucketACL("public-read-write") { t.Fatal("Error:", acl) } - // list all buckets. + // List all buckets. buckets, err := c.ListBuckets() + if len(buckets) == 0 { + t.Fatal("Error: list buckets cannot be empty", buckets) + } if err != nil { t.Fatal("Error:", err) } @@ -413,14 +616,14 @@ func TestFunctional(t *testing.T) { objectName := bucketName + "unique" - // generate data + // Generate data buf := make([]byte, rand.Intn(1<<19)) _, err = io.ReadFull(crand.Reader, buf) if err != nil { t.Fatal("Error: ", err) } - n, err := c.PutObject(bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), "") + n, err := c.PutObject(bucketName, objectName, bytes.NewReader(buf), "") if err != nil { t.Fatal("Error: ", err) } @@ -428,7 +631,7 @@ func TestFunctional(t *testing.T) { t.Fatal("Error: bad length ", n, len(buf)) } - n, err = c.PutObject(bucketName, objectName+"-nolength", bytes.NewReader(buf), -1, "binary/octet-stream") + n, err = c.PutObject(bucketName, objectName+"-nolength", bytes.NewReader(buf), "binary/octet-stream") if err != nil { t.Fatal("Error:", err, bucketName, objectName+"-nolength") } @@ -437,7 +640,34 @@ func TestFunctional(t *testing.T) { t.Fatalf("Error: number of bytes does not match, want %v, got %v\n", len(buf), n) } - newReader, _, err := c.GetObject(bucketName, objectName) + // Instantiate a done channel to close all listing. + doneCh := make(chan struct{}) + defer close(doneCh) + + objFound := false + isRecursive := true // Recursive is true. + for obj := range c.ListObjects(bucketName, objectName, isRecursive, doneCh) { + if obj.Key == objectName { + objFound = true + break + } + } + if !objFound { + t.Fatal("Error: object " + objectName + " not found.") + } + + incompObjNotFound := true + for objIncompl := range c.ListIncompleteUploads(bucketName, objectName, isRecursive, doneCh) { + if objIncompl.Key != "" { + incompObjNotFound = false + break + } + } + if !incompObjNotFound { + t.Fatal("Error: unexpected dangling incomplete upload found.") + } + + newReader, err := c.GetObject(bucketName, objectName) if err != nil { t.Fatal("Error: ", err) } @@ -451,15 +681,7 @@ func TestFunctional(t *testing.T) { t.Fatal("Error: bytes mismatch.") } - n, err = c.FPutObject(bucketName, objectName+"-f", fileName, "text/plain") - if err != nil { - t.Fatal("Error: ", err) - } - if n != totalSize { - t.Fatal("Error: bad length ", n, totalSize) - } - - err = c.FGetObject(bucketName, objectName+"-f", fileName+"-f") + err = c.FGetObject(bucketName, objectName, fileName+"-f") if err != nil { t.Fatal("Error: ", err) } @@ -503,7 +725,7 @@ func TestFunctional(t *testing.T) { t.Fatal("Error: ", err) } - newReader, _, err = c.GetObject(bucketName, objectName+"-presigned") + newReader, err = c.GetObject(bucketName, objectName+"-presigned") if err != nil { t.Fatal("Error: ", err) } @@ -537,11 +759,11 @@ func TestFunctional(t *testing.T) { if err != nil { t.Fatal("Error:", err) } - err = c.RemoveBucket("bucket1") + err = c.RemoveBucket(bucketName) if err == nil { t.Fatal("Error:") } - if err.Error() != "The specified bucket does not exist." { + if err.Error() != "The specified bucket does not exist" { t.Fatal("Error: ", err) } if err = os.Remove(fileName); err != nil { diff --git a/Godeps/_workspace/src/github.com/minio/minio-go/api_private_test.go b/Godeps/_workspace/src/github.com/minio/minio-go/api_unit_test.go similarity index 60% rename from Godeps/_workspace/src/github.com/minio/minio-go/api_private_test.go rename to Godeps/_workspace/src/github.com/minio/minio-go/api_unit_test.go index 2bda99f47..13afcdc45 100644 --- a/Godeps/_workspace/src/github.com/minio/minio-go/api_private_test.go +++ b/Godeps/_workspace/src/github.com/minio/minio-go/api_unit_test.go @@ -17,11 +17,126 @@ package minio import ( + "fmt" + "net/http" "net/url" + "strings" "testing" ) -func TestSignature(t *testing.T) { +func TestEncodeURL2Path(t *testing.T) { + type urlStrings struct { + objName string + encodedObjName string + } + + bucketName := "bucketName" + want := []urlStrings{ + { + objName: "本語", + encodedObjName: "%E6%9C%AC%E8%AA%9E", + }, + { + objName: "本語.1", + encodedObjName: "%E6%9C%AC%E8%AA%9E.1", + }, + { + objName: ">123>3123123", + encodedObjName: "%3E123%3E3123123", + }, + { + objName: "test 1 2.txt", + encodedObjName: "test%201%202.txt", + }, + { + objName: "test++ 1.txt", + encodedObjName: "test%2B%2B%201.txt", + }, + } + + for _, o := range want { + u, err := url.Parse(fmt.Sprintf("https://%s.s3.amazonaws.com/%s", bucketName, o.objName)) + if err != nil { + t.Fatal("Error:", err) + } + urlPath := "/" + bucketName + "/" + o.encodedObjName + if urlPath != encodeURL2Path(u) { + t.Fatal("Error") + } + } +} + +func TestErrorResponse(t *testing.T) { + var err error + err = ErrorResponse{ + Code: "Testing", + } + errResp := ToErrorResponse(err) + if errResp.Code != "Testing" { + t.Fatal("Type conversion failed, we have an empty struct.") + } + + // Test http response decoding. + var httpResponse *http.Response + // Set empty variables + httpResponse = nil + var bucketName, objectName string + + // Should fail with invalid argument. + err = HTTPRespToErrorResponse(httpResponse, bucketName, objectName) + errResp = ToErrorResponse(err) + if errResp.Code != "InvalidArgument" { + t.Fatal("Empty response input should return invalid argument.") + } +} + +func TestSignatureCalculation(t *testing.T) { + req, err := http.NewRequest("GET", "https://s3.amazonaws.com", nil) + if err != nil { + t.Fatal("Error:", err) + } + req = SignV4(*req, "", "", "us-east-1") + if req.Header.Get("Authorization") != "" { + t.Fatal("Error: anonymous credentials should not have Authorization header.") + } + + req = PreSignV4(*req, "", "", "us-east-1", 0) + if strings.Contains(req.URL.RawQuery, "X-Amz-Signature") { + t.Fatal("Error: anonymous credentials should not have Signature query resource.") + } + + req = SignV2(*req, "", "") + if req.Header.Get("Authorization") != "" { + t.Fatal("Error: anonymous credentials should not have Authorization header.") + } + + req = PreSignV2(*req, "", "", 0) + if strings.Contains(req.URL.RawQuery, "Signature") { + t.Fatal("Error: anonymous credentials should not have Signature query resource.") + } + + req = SignV4(*req, "ACCESS-KEY", "SECRET-KEY", "us-east-1") + if req.Header.Get("Authorization") == "" { + t.Fatal("Error: normal credentials should have Authorization header.") + } + + req = PreSignV4(*req, "ACCESS-KEY", "SECRET-KEY", "us-east-1", 0) + if !strings.Contains(req.URL.RawQuery, "X-Amz-Signature") { + t.Fatal("Error: normal credentials should have Signature query resource.") + } + + req = SignV2(*req, "ACCESS-KEY", "SECRET-KEY") + if req.Header.Get("Authorization") == "" { + t.Fatal("Error: normal credentials should have Authorization header.") + } + + req = PreSignV2(*req, "ACCESS-KEY", "SECRET-KEY", 0) + if !strings.Contains(req.URL.RawQuery, "Signature") { + t.Fatal("Error: normal credentials should not have Signature query resource.") + } +} + +func TestSignatureType(t *testing.T) { clnt := Client{} if !clnt.signature.isV4() { t.Fatal("Error") diff --git a/Godeps/_workspace/src/github.com/minio/minio-go/appveyor.yml b/Godeps/_workspace/src/github.com/minio/minio-go/appveyor.yml index 444696bc5..5b8824d45 100644 --- a/Godeps/_workspace/src/github.com/minio/minio-go/appveyor.yml +++ b/Godeps/_workspace/src/github.com/minio/minio-go/appveyor.yml @@ -26,8 +26,8 @@ build_script: - gofmt -s -l . - golint github.com/minio/minio-go... - deadcode - - go test - - go test -test.short -race + - go test -short -v + - go test -short -race -v # to disable automatic tests test: off diff --git a/Godeps/_workspace/src/github.com/minio/minio-go/bucket-acl.go b/Godeps/_workspace/src/github.com/minio/minio-go/bucket-acl.go index 89c386ca1..d8eda0f54 100644 --- a/Godeps/_workspace/src/github.com/minio/minio-go/bucket-acl.go +++ b/Godeps/_workspace/src/github.com/minio/minio-go/bucket-acl.go @@ -16,7 +16,7 @@ package minio -// BucketACL - bucket level access control. +// BucketACL - Bucket level access control. type BucketACL string // Different types of ACL's currently supported for buckets. @@ -35,7 +35,7 @@ func (b BucketACL) String() string { return string(b) } -// isValidBucketACL - is provided acl string supported. +// isValidBucketACL - Is provided acl string supported. func (b BucketACL) isValidBucketACL() bool { switch true { case b.isPrivate(): @@ -47,29 +47,29 @@ func (b BucketACL) isValidBucketACL() bool { case b.isAuthenticated(): return true case b.String() == "private": - // by default its "private" + // By default its "private" return true default: return false } } -// isPrivate - is acl Private. +// isPrivate - Is acl Private. func (b BucketACL) isPrivate() bool { return b == bucketPrivate } -// isPublicRead - is acl PublicRead. +// isPublicRead - Is acl PublicRead. func (b BucketACL) isReadOnly() bool { return b == bucketReadOnly } -// isPublicReadWrite - is acl PublicReadWrite. +// isPublicReadWrite - Is acl PublicReadWrite. func (b BucketACL) isPublic() bool { return b == bucketPublic } -// isAuthenticated - is acl AuthenticatedRead. +// isAuthenticated - Is acl AuthenticatedRead. func (b BucketACL) isAuthenticated() bool { return b == bucketAuthenticated } diff --git a/Godeps/_workspace/src/github.com/minio/minio-go/bucket-cache.go b/Godeps/_workspace/src/github.com/minio/minio-go/bucket-cache.go index d0993ba4a..849aed9fe 100644 --- a/Godeps/_workspace/src/github.com/minio/minio-go/bucket-cache.go +++ b/Godeps/_workspace/src/github.com/minio/minio-go/bucket-cache.go @@ -24,25 +24,26 @@ import ( "sync" ) -// bucketLocationCache provides simple mechansim to hold bucket locations in memory. +// bucketLocationCache - Provides simple mechansim to hold bucket +// locations in memory. type bucketLocationCache struct { - // Mutex is used for handling the concurrent - // read/write requests for cache + // mutex is used for handling the concurrent + // read/write requests for cache. sync.RWMutex // items holds the cached bucket locations. items map[string]string } -// newBucketLocationCache provides a new bucket location cache to be used -// internally with the client object. +// newBucketLocationCache - Provides a new bucket location cache to be +// used internally with the client object. func newBucketLocationCache() *bucketLocationCache { return &bucketLocationCache{ items: make(map[string]string), } } -// Get returns a value of a given key if it exists +// Get - Returns a value of a given key if it exists. func (r *bucketLocationCache) Get(bucketName string) (location string, ok bool) { r.RLock() defer r.RUnlock() @@ -50,21 +51,21 @@ func (r *bucketLocationCache) Get(bucketName string) (location string, ok bool) return } -// Set will persist a value to the cache +// Set - Will persist a value into cache. func (r *bucketLocationCache) Set(bucketName string, location string) { r.Lock() defer r.Unlock() r.items[bucketName] = location } -// Delete deletes a bucket name. +// Delete - Deletes a bucket name from cache. func (r *bucketLocationCache) Delete(bucketName string) { r.Lock() defer r.Unlock() delete(r.items, bucketName) } -// getBucketLocation - get location for the bucketName from location map cache. +// getBucketLocation - Get location for the bucketName from location map cache. func (c Client) getBucketLocation(bucketName string) (string, error) { // For anonymous requests, default to "us-east-1" and let other calls // move forward. @@ -101,12 +102,12 @@ func (c Client) getBucketLocation(bucketName string) (string, error) { } location := locationConstraint - // location is empty will be 'us-east-1'. + // Location is empty will be 'us-east-1'. if location == "" { location = "us-east-1" } - // location can be 'EU' convert it to meaningful 'eu-west-1'. + // Location can be 'EU' convert it to meaningful 'eu-west-1'. if location == "EU" { location = "eu-west-1" } @@ -118,7 +119,7 @@ func (c Client) getBucketLocation(bucketName string) (string, error) { return location, nil } -// getBucketLocationRequest wrapper creates a new getBucketLocation request. +// getBucketLocationRequest - Wrapper creates a new getBucketLocation request. func (c Client) getBucketLocationRequest(bucketName string) (*http.Request, error) { // Set location query. urlValues := make(url.Values) @@ -129,16 +130,16 @@ func (c Client) getBucketLocationRequest(bucketName string) (*http.Request, erro targetURL.Path = filepath.Join(bucketName, "") targetURL.RawQuery = urlValues.Encode() - // get a new HTTP request for the method. + // Get a new HTTP request for the method. req, err := http.NewRequest("GET", targetURL.String(), nil) if err != nil { return nil, err } - // set UserAgent for the request. + // Set UserAgent for the request. c.setUserAgent(req) - // set sha256 sum for signature calculation only with signature version '4'. + // Set sha256 sum for signature calculation only with signature version '4'. if c.signature.isV4() { req.Header.Set("X-Amz-Content-Sha256", hex.EncodeToString(sum256([]byte{}))) } diff --git a/Godeps/_workspace/src/github.com/minio/minio-go/common-methods.go b/Godeps/_workspace/src/github.com/minio/minio-go/common-methods.go deleted file mode 100644 index 636e06f6f..000000000 --- a/Godeps/_workspace/src/github.com/minio/minio-go/common-methods.go +++ /dev/null @@ -1,52 +0,0 @@ -/* - * Minio Go Library for Amazon S3 Compatible Cloud Storage (C) 2015 Minio, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package minio - -import ( - "crypto/hmac" - "crypto/md5" - "crypto/sha256" - "encoding/xml" - "io" -) - -// xmlDecoder provide decoded value in xml. -func xmlDecoder(body io.Reader, v interface{}) error { - d := xml.NewDecoder(body) - return d.Decode(v) -} - -// sum256 calculate sha256 sum for an input byte array. -func sum256(data []byte) []byte { - hash := sha256.New() - hash.Write(data) - return hash.Sum(nil) -} - -// sumMD5 calculate md5 sum for an input byte array. -func sumMD5(data []byte) []byte { - hash := md5.New() - hash.Write(data) - return hash.Sum(nil) -} - -// sumHMAC calculate hmac between two input byte array. -func sumHMAC(key []byte, data []byte) []byte { - hash := hmac.New(sha256.New, key) - hash.Write(data) - return hash.Sum(nil) -} diff --git a/Godeps/_workspace/src/github.com/minio/minio-go/constants.go b/Godeps/_workspace/src/github.com/minio/minio-go/constants.go index f4978019f..c97803b8d 100644 --- a/Godeps/_workspace/src/github.com/minio/minio-go/constants.go +++ b/Godeps/_workspace/src/github.com/minio/minio-go/constants.go @@ -25,14 +25,18 @@ const minimumPartSize = 1024 * 1024 * 5 // maxParts - maximum parts for a single multipart session. const maxParts = 10000 -// maxPartSize - maximum part size 5GiB for a single multipart upload operation. +// maxPartSize - maximum part size 5GiB for a single multipart upload +// operation. const maxPartSize = 1024 * 1024 * 1024 * 5 -// maxSinglePutObjectSize - maximum size 5GiB of object per PUT operation. +// maxSinglePutObjectSize - maximum size 5GiB of object per PUT +// operation. const maxSinglePutObjectSize = 1024 * 1024 * 1024 * 5 -// maxMultipartPutObjectSize - maximum size 5TiB of object for Multipart operation. +// maxMultipartPutObjectSize - maximum size 5TiB of object for +// Multipart operation. const maxMultipartPutObjectSize = 1024 * 1024 * 1024 * 1024 * 5 -// optimalReadAtBufferSize - optimal buffer 5MiB used for reading through ReadAt operation. +// optimalReadAtBufferSize - optimal buffer 5MiB used for reading +// through ReadAt operation. const optimalReadAtBufferSize = 1024 * 1024 * 5 diff --git a/Godeps/_workspace/src/github.com/minio/minio-go/examples/play/getobject.go b/Godeps/_workspace/src/github.com/minio/minio-go/examples/play/getobject.go index 041a136c1..de0b12cc3 100644 --- a/Godeps/_workspace/src/github.com/minio/minio-go/examples/play/getobject.go +++ b/Godeps/_workspace/src/github.com/minio/minio-go/examples/play/getobject.go @@ -29,28 +29,40 @@ import ( func main() { // Note: my-bucketname, my-objectname and my-testfile are dummy values, please replace them with original values. - // Requests are always secure by default. set inSecure=true to enable insecure access. - // inSecure boolean is the last argument for New(). + // Requests are always secure (HTTPS) by default. Set insecure=true to enable insecure (HTTP) access. + // This boolean value is the last argument for New(). - // New provides a client object backend by automatically detected signature type based - // on the provider. + // New returns an Amazon S3 compatible client object. API copatibality (v2 or v4) is automatically + // determined based on the Endpoint value. s3Client, err := minio.New("play.minio.io:9002", "Q3AM3UQ867SPQQA43P2F", "zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG", false) if err != nil { log.Fatalln(err) } - reader, _, err := s3Client.GetObject("my-bucketname", "my-objectname") + reader, err := s3Client.GetObject("my-bucketname", "my-objectname") if err != nil { log.Fatalln(err) } + defer reader.Close() - localfile, err := os.Create("my-testfile") + reader, err := s3Client.GetObject("my-bucketname", "my-objectname") + if err != nil { + log.Fatalln(err) + } + defer reader.Close() + + localFile, err := os.Create("my-testfile") if err != nil { log.Fatalln(err) } defer localfile.Close() - if _, err = io.Copy(localfile, reader); err != nil { + stat, err := reader.Stat() + if err != nil { + log.Fatalln(err) + } + + if _, err := io.CopyN(localFile, reader, stat.Size); err != nil { log.Fatalln(err) } } diff --git a/Godeps/_workspace/src/github.com/minio/minio-go/examples/play/getobjectpartial.go b/Godeps/_workspace/src/github.com/minio/minio-go/examples/play/getobjectpartial.go deleted file mode 100644 index db65359ca..000000000 --- a/Godeps/_workspace/src/github.com/minio/minio-go/examples/play/getobjectpartial.go +++ /dev/null @@ -1,91 +0,0 @@ -// +build ignore - -/* - * Minio Go Library for Amazon S3 Compatible Cloud Storage (C) 2015 Minio, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package main - -import ( - "errors" - "io" - "log" - "os" - - "github.com/minio/minio-go" -) - -func main() { - // Note: my-bucketname, my-objectname and my-testfile are dummy values, please replace them with original values. - - // Requests are always secure (HTTPS) by default. Set insecure=true to enable insecure (HTTP) access. - // This boolean value is the last argument for New(). - - // New returns an Amazon S3 compatible client object. API copatibality (v2 or v4) is automatically - // determined based on the Endpoint value. - s3Client, err := minio.New("play.minio.io:9002", "Q3AM3UQ867SPQQA43P2F", "zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG", false) - if err != nil { - log.Fatalln(err) - } - - reader, stat, err := s3Client.GetObjectPartial("my-bucketname", "my-objectname") - if err != nil { - log.Fatalln(err) - } - defer reader.Close() - - localFile, err := os.OpenFile("my-testfile", os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0600) - if err != nil { - log.Fatalln(err) - } - defer localfile.Close() - - st, err := localFile.Stat() - if err != nil { - log.Fatalln(err) - } - - readAtOffset := st.Size() - readAtBuffer := make([]byte, 5*1024*1024) - - // Loop and write. - for { - readAtSize, rerr := reader.ReadAt(readAtBuffer, readAtOffset) - if rerr != nil { - if rerr != io.EOF { - log.Fatalln(rerr) - } - } - writeSize, werr := localFile.Write(readAtBuffer[:readAtSize]) - if werr != nil { - log.Fatalln(werr) - } - if readAtSize != writeSize { - log.Fatalln(errors.New("Something really bad happened here.")) - } - readAtOffset += int64(writeSize) - if rerr == io.EOF { - break - } - } - - // totalWritten size. - totalWritten := readAtOffset - - // If found mismatch error out. - if totalWritten != stat.Size { - log.Fatalln(errors.New("Something really bad happened here.")) - } -} diff --git a/Godeps/_workspace/src/github.com/minio/minio-go/examples/play/putobject.go b/Godeps/_workspace/src/github.com/minio/minio-go/examples/play/putobject.go index d7efb7b43..073f75870 100644 --- a/Godeps/_workspace/src/github.com/minio/minio-go/examples/play/putobject.go +++ b/Godeps/_workspace/src/github.com/minio/minio-go/examples/play/putobject.go @@ -44,8 +44,7 @@ func main() { } defer object.Close() - st, _ := object.Stat() - n, err := s3Client.PutObject("my-bucketname", "my-objectname", object, st.Size(), "application/octet-stream") + n, err := s3Client.PutObject("my-bucketname", "my-objectname", object, "application/octet-stream") if err != nil { log.Fatalln(err) } diff --git a/Godeps/_workspace/src/github.com/minio/minio-go/examples/play/putobjectpartial.go b/Godeps/_workspace/src/github.com/minio/minio-go/examples/play/putobjectpartial.go deleted file mode 100644 index aff67f8e9..000000000 --- a/Godeps/_workspace/src/github.com/minio/minio-go/examples/play/putobjectpartial.go +++ /dev/null @@ -1,56 +0,0 @@ -// +build ignore - -/* - * Minio Go Library for Amazon S3 Compatible Cloud Storage (C) 2015 Minio, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package main - -import ( - "log" - "os" - - "github.com/minio/minio-go" -) - -func main() { - // Note: my-bucketname, my-objectname and my-testfile are dummy values, please replace them with original values. - - // Requests are always secure (HTTPS) by default. Set insecure=true to enable insecure (HTTP) access. - // This boolean value is the last argument for New(). - - // New returns an Amazon S3 compatible client object. API copatibality (v2 or v4) is automatically - // determined based on the Endpoint value. - s3Client, err := minio.New("play.minio.io:9002", "Q3AM3UQ867SPQQA43P2F", "zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG", false) - if err != nil { - log.Fatalln(err) - } - - localFile, err := os.Open("testfile") - if err != nil { - log.Fatalln(err) - } - - st, err := localFile.Stat() - if err != nil { - log.Fatalln(err) - } - defer localFile.Close() - - _, err = s3Client.PutObjectPartial("bucket-name", "objectName", localFile, st.Size(), "text/plain") - if err != nil { - log.Fatalln(err) - } -} diff --git a/Godeps/_workspace/src/github.com/minio/minio-go/examples/s3/getobject.go b/Godeps/_workspace/src/github.com/minio/minio-go/examples/s3/getobject.go index 0125491ab..9413dc5e5 100644 --- a/Godeps/_workspace/src/github.com/minio/minio-go/examples/s3/getobject.go +++ b/Godeps/_workspace/src/github.com/minio/minio-go/examples/s3/getobject.go @@ -35,23 +35,29 @@ func main() { // New returns an Amazon S3 compatible client object. API copatibality (v2 or v4) is automatically // determined based on the Endpoint value. - s3Client, err := minio.New("s3.amazonaws.com", "YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", false) + s3Client, err := minio.New("s3.amazonaws.com", "YOUR-ACCESS-KEY-HERE", "YOUR-SECRET-KEY-HERE", false) if err != nil { log.Fatalln(err) } - reader, _, err := s3Client.GetObject("my-bucketname", "my-objectname") + reader, err := s3Client.GetObject("my-bucketname", "my-objectname") if err != nil { log.Fatalln(err) } + defer reader.Close() - localfile, err := os.Create("my-testfile") + localFile, err := os.Create("my-testfile") if err != nil { log.Fatalln(err) } defer localfile.Close() - if _, err = io.Copy(localfile, reader); err != nil { + stat, err := reader.Stat() + if err != nil { + log.Fatalln(err) + } + + if _, err := io.CopyN(localFile, reader, stat.Size); err != nil { log.Fatalln(err) } } diff --git a/Godeps/_workspace/src/github.com/minio/minio-go/examples/s3/getobjectpartial.go b/Godeps/_workspace/src/github.com/minio/minio-go/examples/s3/getobjectpartial.go deleted file mode 100644 index 2c32c8449..000000000 --- a/Godeps/_workspace/src/github.com/minio/minio-go/examples/s3/getobjectpartial.go +++ /dev/null @@ -1,92 +0,0 @@ -// +build ignore - -/* - * Minio Go Library for Amazon S3 Compatible Cloud Storage (C) 2015 Minio, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package main - -import ( - "errors" - "io" - "log" - "os" - - "github.com/minio/minio-go" -) - -func main() { - // Note: YOUR-ACCESSKEYID, YOUR-SECRETACCESSKEY, my-bucketname, my-objectname and - // my-testfile are dummy values, please replace them with original values. - - // Requests are always secure (HTTPS) by default. Set insecure=true to enable insecure (HTTP) access. - // This boolean value is the last argument for New(). - - // New returns an Amazon S3 compatible client object. API copatibality (v2 or v4) is automatically - // determined based on the Endpoint value. - s3Client, err := minio.New("s3.amazonaws.com", "YOUR-ACCESS-KEY-HERE", "YOUR-SECRET-KEY-HERE", false) - if err != nil { - log.Fatalln(err) - } - - reader, stat, err := s3Client.GetObjectPartial("my-bucketname", "my-objectname") - if err != nil { - log.Fatalln(err) - } - defer reader.Close() - - localFile, err := os.OpenFile("my-testfile", os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0600) - if err != nil { - log.Fatalln(err) - } - defer localfile.Close() - - st, err := localFile.Stat() - if err != nil { - log.Fatalln(err) - } - - readAtOffset := st.Size() - readAtBuffer := make([]byte, 5*1024*1024) - - // For loop. - for { - readAtSize, rerr := reader.ReadAt(readAtBuffer, readAtOffset) - if rerr != nil { - if rerr != io.EOF { - log.Fatalln(rerr) - } - } - writeSize, werr := localFile.Write(readAtBuffer[:readAtSize]) - if werr != nil { - log.Fatalln(werr) - } - if readAtSize != writeSize { - log.Fatalln(errors.New("Something really bad happened here.")) - } - readAtOffset += int64(writeSize) - if rerr == io.EOF { - break - } - } - - // totalWritten size. - totalWritten := readAtOffset - - // If found mismatch error out. - if totalWritten != stat.Size { - log.Fatalln(errors.New("Something really bad happened here.")) - } -} diff --git a/Godeps/_workspace/src/github.com/minio/minio-go/examples/s3/putobject.go b/Godeps/_workspace/src/github.com/minio/minio-go/examples/s3/putobject.go index 963060487..2ba90a697 100644 --- a/Godeps/_workspace/src/github.com/minio/minio-go/examples/s3/putobject.go +++ b/Godeps/_workspace/src/github.com/minio/minio-go/examples/s3/putobject.go @@ -45,8 +45,7 @@ func main() { } defer object.Close() - st, _ := object.Stat() - n, err := s3Client.PutObject("my-bucketname", "my-objectname", object, st.Size(), "application/octet-stream") + n, err := s3Client.PutObject("my-bucketname", "my-objectname", object, "application/octet-stream") if err != nil { log.Fatalln(err) } diff --git a/Godeps/_workspace/src/github.com/minio/minio-go/examples/s3/putobjectpartial.go b/Godeps/_workspace/src/github.com/minio/minio-go/examples/s3/putobjectpartial.go deleted file mode 100644 index e59b2ad4d..000000000 --- a/Godeps/_workspace/src/github.com/minio/minio-go/examples/s3/putobjectpartial.go +++ /dev/null @@ -1,57 +0,0 @@ -// +build ignore - -/* - * Minio Go Library for Amazon S3 Compatible Cloud Storage (C) 2015 Minio, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package main - -import ( - "log" - "os" - - "github.com/minio/minio-go" -) - -func main() { - // Note: YOUR-ACCESSKEYID, YOUR-SECRETACCESSKEY, my-bucketname, my-objectname and - // my-testfile are dummy values, please replace them with original values. - - // Requests are always secure (HTTPS) by default. Set insecure=true to enable insecure (HTTP) access. - // This boolean value is the last argument for New(). - - // New returns an Amazon S3 compatible client object. API copatibality (v2 or v4) is automatically - // determined based on the Endpoint value. - s3Client, err := minio.New("s3.amazonaws.com", "YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", false) - if err != nil { - log.Fatalln(err) - } - - localFile, err := os.Open("my-testfile") - if err != nil { - log.Fatalln(err) - } - - st, err := localFile.Stat() - if err != nil { - log.Fatalln(err) - } - defer localFile.Close() - - _, err = s3Client.PutObjectPartial("my-bucketname", "my-objectname", localFile, st.Size(), "text/plain") - if err != nil { - log.Fatalln(err) - } -} diff --git a/Godeps/_workspace/src/github.com/minio/minio-go/post-policy.go b/Godeps/_workspace/src/github.com/minio/minio-go/post-policy.go index 2d3082755..2a675d770 100644 --- a/Godeps/_workspace/src/github.com/minio/minio-go/post-policy.go +++ b/Godeps/_workspace/src/github.com/minio/minio-go/post-policy.go @@ -2,7 +2,6 @@ package minio import ( "encoding/base64" - "errors" "fmt" "strings" "time" @@ -11,7 +10,8 @@ import ( // expirationDateFormat date format for expiration key in json policy. const expirationDateFormat = "2006-01-02T15:04:05.999Z" -// policyCondition explanation: http://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-HTTPPOSTConstructPolicy.html +// policyCondition explanation: +// http://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-HTTPPOSTConstructPolicy.html // // Example: // @@ -27,11 +27,15 @@ type policyCondition struct { value string } -// PostPolicy provides strict static type conversion and validation for Amazon S3's POST policy JSON string. +// PostPolicy - Provides strict static type conversion and validation +// for Amazon S3's POST policy JSON string. type PostPolicy struct { - expiration time.Time // expiration date and time of the POST policy. - conditions []policyCondition // collection of different policy conditions. - // contentLengthRange minimum and maximum allowable size for the uploaded content. + // Expiration date and time of the POST policy. + expiration time.Time + // Collection of different policy conditions. + conditions []policyCondition + // ContentLengthRange minimum and maximum allowable size for the + // uploaded content. contentLengthRange struct { min int64 max int64 @@ -41,7 +45,7 @@ type PostPolicy struct { formData map[string]string } -// NewPostPolicy instantiate new post policy. +// NewPostPolicy - Instantiate new post policy. func NewPostPolicy() *PostPolicy { p := &PostPolicy{} p.conditions = make([]policyCondition, 0) @@ -49,19 +53,19 @@ func NewPostPolicy() *PostPolicy { return p } -// SetExpires expiration time. +// SetExpires - Sets expiration time for the new policy. func (p *PostPolicy) SetExpires(t time.Time) error { if t.IsZero() { - return errors.New("No expiry time set.") + return ErrInvalidArgument("No expiry time set.") } p.expiration = t return nil } -// SetKey Object name. +// SetKey - Sets an object name for the policy based upload. func (p *PostPolicy) SetKey(key string) error { if strings.TrimSpace(key) == "" || key == "" { - return errors.New("Object name is not specified.") + return ErrInvalidArgument("Object name is empty.") } policyCond := policyCondition{ matchType: "eq", @@ -75,10 +79,11 @@ func (p *PostPolicy) SetKey(key string) error { return nil } -// SetKeyStartsWith Object name that can start with. +// SetKeyStartsWith - Sets an object name that an policy based upload +// can start with. func (p *PostPolicy) SetKeyStartsWith(keyStartsWith string) error { if strings.TrimSpace(keyStartsWith) == "" || keyStartsWith == "" { - return errors.New("Object prefix is not specified.") + return ErrInvalidArgument("Object prefix is empty.") } policyCond := policyCondition{ matchType: "starts-with", @@ -92,10 +97,10 @@ func (p *PostPolicy) SetKeyStartsWith(keyStartsWith string) error { return nil } -// SetBucket bucket name. +// SetBucket - Sets bucket at which objects will be uploaded to. func (p *PostPolicy) SetBucket(bucketName string) error { if strings.TrimSpace(bucketName) == "" || bucketName == "" { - return errors.New("Bucket name is not specified.") + return ErrInvalidArgument("Bucket name is empty.") } policyCond := policyCondition{ matchType: "eq", @@ -109,10 +114,11 @@ func (p *PostPolicy) SetBucket(bucketName string) error { return nil } -// SetContentType content-type. +// SetContentType - Sets content-type of the object for this policy +// based upload. func (p *PostPolicy) SetContentType(contentType string) error { if strings.TrimSpace(contentType) == "" || contentType == "" { - return errors.New("No content type specified.") + return ErrInvalidArgument("No content type specified.") } policyCond := policyCondition{ matchType: "eq", @@ -126,16 +132,17 @@ func (p *PostPolicy) SetContentType(contentType string) error { return nil } -// SetContentLengthRange - set new min and max content length condition. +// SetContentLengthRange - Set new min and max content length +// condition for all incoming uploads. func (p *PostPolicy) SetContentLengthRange(min, max int64) error { if min > max { - return errors.New("minimum limit is larger than maximum limit") + return ErrInvalidArgument("Minimum limit is larger than maximum limit.") } if min < 0 { - return errors.New("minimum limit cannot be negative") + return ErrInvalidArgument("Minimum limit cannot be negative.") } if max < 0 { - return errors.New("maximum limit cannot be negative") + return ErrInvalidArgument("Maximum limit cannot be negative.") } p.contentLengthRange.min = min p.contentLengthRange.max = max @@ -145,18 +152,18 @@ func (p *PostPolicy) SetContentLengthRange(min, max int64) error { // addNewPolicy - internal helper to validate adding new policies. func (p *PostPolicy) addNewPolicy(policyCond policyCondition) error { if policyCond.matchType == "" || policyCond.condition == "" || policyCond.value == "" { - return errors.New("Policy fields empty.") + return ErrInvalidArgument("Policy fields are empty.") } p.conditions = append(p.conditions, policyCond) return nil } -// Stringer interface for printing in pretty manner. +// Stringer interface for printing policy in json formatted string. func (p PostPolicy) String() string { return string(p.marshalJSON()) } -// marshalJSON provides Marshalled JSON. +// marshalJSON - Provides Marshalled JSON in bytes. func (p PostPolicy) marshalJSON() []byte { expirationStr := `"expiration":"` + p.expiration.Format(expirationDateFormat) + `"` var conditionsStr string @@ -178,7 +185,7 @@ func (p PostPolicy) marshalJSON() []byte { return []byte(retStr) } -// base64 produces base64 of PostPolicy's Marshalled json. +// base64 - Produces base64 of PostPolicy's Marshalled json. func (p PostPolicy) base64() string { return base64.StdEncoding.EncodeToString(p.marshalJSON()) } diff --git a/Godeps/_workspace/src/github.com/minio/minio-go/request-signature-v2.go b/Godeps/_workspace/src/github.com/minio/minio-go/request-signature-v2.go index 956b04f23..055fd8598 100644 --- a/Godeps/_workspace/src/github.com/minio/minio-go/request-signature-v2.go +++ b/Godeps/_workspace/src/github.com/minio/minio-go/request-signature-v2.go @@ -30,7 +30,7 @@ import ( "time" ) -// signature and API related constants. +// Signature and API related constants. const ( signV2Algorithm = "AWS" ) @@ -55,14 +55,14 @@ func encodeURL2Path(u *url.URL) (path string) { } // PreSignV2 - presign the request in following style. -// https://${S3_BUCKET}.s3.amazonaws.com/${S3_OBJECT}?AWSAccessKeyId=${S3_ACCESS_KEY}&Expires=${TIMESTAMP}&Signature=${SIGNATURE} +// https://${S3_BUCKET}.s3.amazonaws.com/${S3_OBJECT}?AWSAccessKeyId=${S3_ACCESS_KEY}&Expires=${TIMESTAMP}&Signature=${SIGNATURE}. func PreSignV2(req http.Request, accessKeyID, secretAccessKey string, expires int64) *http.Request { - // presign is a noop for anonymous credentials. + // Presign is not needed for anonymous credentials. if accessKeyID == "" || secretAccessKey == "" { - return nil + return &req } d := time.Now().UTC() - // Add date if not present + // Add date if not present. if date := req.Header.Get("Date"); date == "" { req.Header.Set("Date", d.Format(http.TimeFormat)) } @@ -73,12 +73,12 @@ func PreSignV2(req http.Request, accessKeyID, secretAccessKey string, expires in // Find epoch expires when the request will expire. epochExpires := d.Unix() + expires - // get string to sign. + // Get string to sign. stringToSign := fmt.Sprintf("%s\n\n\n%d\n%s", req.Method, epochExpires, path) hm := hmac.New(sha1.New, []byte(secretAccessKey)) hm.Write([]byte(stringToSign)) - // calculate signature. + // Calculate signature. signature := base64.StdEncoding.EncodeToString(hm.Sum(nil)) query := req.URL.Query() @@ -98,7 +98,8 @@ func PreSignV2(req http.Request, accessKeyID, secretAccessKey string, expires in return &req } -// PostPresignSignatureV2 - presigned signature for PostPolicy request +// PostPresignSignatureV2 - presigned signature for PostPolicy +// request. func PostPresignSignatureV2(policyBase64, secretAccessKey string) string { hm := hmac.New(sha1.New, []byte(secretAccessKey)) hm.Write([]byte(policyBase64)) @@ -124,6 +125,11 @@ func PostPresignSignatureV2(policyBase64, secretAccessKey string) string { // SignV2 sign the request before Do() (AWS Signature Version 2). func SignV2(req http.Request, accessKeyID, secretAccessKey string) *http.Request { + // Signature calculation is not needed for anonymous credentials. + if accessKeyID == "" || secretAccessKey == "" { + return &req + } + // Initial time. d := time.Now().UTC() @@ -160,11 +166,11 @@ func SignV2(req http.Request, accessKeyID, secretAccessKey string) *http.Request // CanonicalizedResource; func getStringToSignV2(req http.Request) string { buf := new(bytes.Buffer) - // write standard headers. + // Write standard headers. writeDefaultHeaders(buf, req) - // write canonicalized protocol headers if any. + // Write canonicalized protocol headers if any. writeCanonicalizedHeaders(buf, req) - // write canonicalized Query resources if any. + // Write canonicalized Query resources if any. writeCanonicalizedResource(buf, req) return buf.String() } @@ -186,7 +192,7 @@ func writeCanonicalizedHeaders(buf *bytes.Buffer, req http.Request) { var protoHeaders []string vals := make(map[string][]string) for k, vv := range req.Header { - // all the AMZ and GOOG headers should be lowercase + // All the AMZ headers should be lowercase lk := strings.ToLower(k) if strings.HasPrefix(lk, "x-amz") { protoHeaders = append(protoHeaders, lk) @@ -246,6 +252,7 @@ var resourceList = []string{ // + // [ sub-resource, if present. For example "?acl", "?location", "?logging", or "?torrent"]; func writeCanonicalizedResource(buf *bytes.Buffer, req http.Request) error { + // Save request URL. requestURL := req.URL // Get encoded URL path. @@ -256,20 +263,21 @@ func writeCanonicalizedResource(buf *bytes.Buffer, req http.Request) error { if requestURL.RawQuery != "" { var n int vals, _ := url.ParseQuery(requestURL.RawQuery) - // loop through all the supported resourceList. + // Verify if any sub resource queries are present, if yes + // canonicallize them. for _, resource := range resourceList { if vv, ok := vals[resource]; ok && len(vv) > 0 { n++ - // first element + // First element switch n { case 1: buf.WriteByte('?') - // the rest + // The rest default: buf.WriteByte('&') } buf.WriteString(resource) - // request parameters + // Request parameters if len(vv[0]) > 0 { buf.WriteByte('=') buf.WriteString(url.QueryEscape(vv[0])) diff --git a/Godeps/_workspace/src/github.com/minio/minio-go/request-signature-v4.go b/Godeps/_workspace/src/github.com/minio/minio-go/request-signature-v4.go index 515d8ab18..27c292a55 100644 --- a/Godeps/_workspace/src/github.com/minio/minio-go/request-signature-v4.go +++ b/Godeps/_workspace/src/github.com/minio/minio-go/request-signature-v4.go @@ -26,7 +26,7 @@ import ( "time" ) -// signature and API related constants. +// Signature and API related constants. const ( signV4Algorithm = "AWS4-HMAC-SHA256" iso8601DateFormat = "20060102T150405Z" @@ -34,28 +34,35 @@ const ( ) /// -/// Excerpts from @lsegal - https://github.com/aws/aws-sdk-js/issues/659#issuecomment-120477258. +/// Excerpts from @lsegal - +/// https://github.com/aws/aws-sdk-js/issues/659#issuecomment-120477258. /// /// User-Agent: /// -/// This is ignored from signing because signing this causes problems with generating pre-signed URLs -/// (that are executed by other agents) or when customers pass requests through proxies, which may -/// modify the user-agent. +/// This is ignored from signing because signing this causes +/// problems with generating pre-signed URLs (that are executed +/// by other agents) or when customers pass requests through +/// proxies, which may modify the user-agent. /// /// Content-Length: /// -/// This is ignored from signing because generating a pre-signed URL should not provide a content-length -/// constraint, specifically when vending a S3 pre-signed PUT URL. The corollary to this is that when -/// sending regular requests (non-pre-signed), the signature contains a checksum of the body, which -/// implicitly validates the payload length (since changing the number of bytes would change the checksum) +/// This is ignored from signing because generating a pre-signed +/// URL should not provide a content-length constraint, +/// specifically when vending a S3 pre-signed PUT URL. The +/// corollary to this is that when sending regular requests +/// (non-pre-signed), the signature contains a checksum of the +/// body, which implicitly validates the payload length (since +/// changing the number of bytes would change the checksum) /// and therefore this header is not valuable in the signature. /// /// Content-Type: /// -/// Signing this header causes quite a number of problems in browser environments, where browsers -/// like to modify and normalize the content-type header in different ways. There is more information -/// on this in https://github.com/aws/aws-sdk-js/issues/244. Avoiding this field simplifies logic -/// and reduces the possibility of future bugs +/// Signing this header causes quite a number of problems in +/// browser environments, where browsers like to modify and +/// normalize the content-type header in different ways. There is +/// more information on this in https://goo.gl/2E9gyy. Avoiding +/// this field simplifies logic and reduces the possibility of +/// future bugs. /// /// Authorization: /// @@ -68,7 +75,7 @@ var ignoredHeaders = map[string]bool{ "User-Agent": true, } -// getSigningKey hmac seed to calculate final signature +// getSigningKey hmac seed to calculate final signature. func getSigningKey(secret, loc string, t time.Time) []byte { date := sumHMAC([]byte("AWS4"+secret), []byte(t.Format(yyyymmdd))) location := sumHMAC(date, []byte(loc)) @@ -77,12 +84,13 @@ func getSigningKey(secret, loc string, t time.Time) []byte { return signingKey } -// getSignature final signature in hexadecimal form +// getSignature final signature in hexadecimal form. func getSignature(signingKey []byte, stringToSign string) string { return hex.EncodeToString(sumHMAC(signingKey, []byte(stringToSign))) } -// getScope generate a string of a specific date, an AWS region, and a service +// getScope generate a string of a specific date, an AWS region, and a +// service. func getScope(location string, t time.Time) string { scope := strings.Join([]string{ t.Format(yyyymmdd), @@ -93,13 +101,14 @@ func getScope(location string, t time.Time) string { return scope } -// getCredential generate a credential string +// getCredential generate a credential string. func getCredential(accessKeyID, location string, t time.Time) string { scope := getScope(location, t) return accessKeyID + "/" + scope } -// getHashedPayload get the hexadecimal value of the SHA256 hash of the request payload +// getHashedPayload get the hexadecimal value of the SHA256 hash of +// the request payload. func getHashedPayload(req http.Request) string { hashedPayload := req.Header.Get("X-Amz-Content-Sha256") if hashedPayload == "" { @@ -109,7 +118,8 @@ func getHashedPayload(req http.Request) string { return hashedPayload } -// getCanonicalHeaders generate a list of request headers for signature. +// getCanonicalHeaders generate a list of request headers for +// signature. func getCanonicalHeaders(req http.Request) string { var headers []string vals := make(map[string][]string) @@ -124,6 +134,8 @@ func getCanonicalHeaders(req http.Request) string { sort.Strings(headers) var buf bytes.Buffer + // Save all the headers in canonical form
: newline + // separated for each header. for _, k := range headers { buf.WriteString(k) buf.WriteByte(':') @@ -145,12 +157,13 @@ func getCanonicalHeaders(req http.Request) string { } // getSignedHeaders generate all signed request headers. -// i.e alphabetically sorted, semicolon-separated list of lowercase request header names +// i.e lexically sorted, semicolon-separated list of lowercase +// request header names. func getSignedHeaders(req http.Request) string { var headers []string for k := range req.Header { if _, ok := ignoredHeaders[http.CanonicalHeaderKey(k)]; ok { - continue // ignored header + continue // Ignored header found continue. } headers = append(headers, strings.ToLower(k)) } @@ -168,7 +181,6 @@ func getSignedHeaders(req http.Request) string { // \n // \n // -// func getCanonicalRequest(req http.Request) string { req.URL.RawQuery = strings.Replace(req.URL.Query().Encode(), "+", "%20", -1) canonicalRequest := strings.Join([]string{ @@ -193,20 +205,21 @@ func getStringToSignV4(t time.Time, location, canonicalRequest string) string { // PreSignV4 presign the request, in accordance with // http://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-query-string-auth.html. func PreSignV4(req http.Request, accessKeyID, secretAccessKey, location string, expires int64) *http.Request { - // presign is a noop for anonymous credentials. + // Presign is not needed for anonymous credentials. if accessKeyID == "" || secretAccessKey == "" { - return nil + return &req } + // Initial time. t := time.Now().UTC() - // get credential string. + // Get credential string. credential := getCredential(accessKeyID, location, t) // Get all signed headers. signedHeaders := getSignedHeaders(req) - // set URL query. + // Set URL query. query := req.URL.Query() query.Set("X-Amz-Algorithm", signV4Algorithm) query.Set("X-Amz-Date", t.Format(iso8601DateFormat)) @@ -221,10 +234,10 @@ func PreSignV4(req http.Request, accessKeyID, secretAccessKey, location string, // Get string to sign from canonical request. stringToSign := getStringToSignV4(t, location, canonicalRequest) - // get hmac signing key. + // Gext hmac signing key. signingKey := getSigningKey(secretAccessKey, location, t) - // calculate signature. + // Calculate signature. signature := getSignature(signingKey, stringToSign) // Add signature header to RawQuery. @@ -233,9 +246,12 @@ func PreSignV4(req http.Request, accessKeyID, secretAccessKey, location string, return &req } -// PostPresignSignatureV4 - presigned signature for PostPolicy requests. +// PostPresignSignatureV4 - presigned signature for PostPolicy +// requests. func PostPresignSignatureV4(policyBase64 string, t time.Time, secretAccessKey, location string) string { + // Get signining key. signingkey := getSigningKey(secretAccessKey, location, t) + // Calculate signature. signature := getSignature(signingkey, policyBase64) return signature } @@ -243,6 +259,11 @@ func PostPresignSignatureV4(policyBase64 string, t time.Time, secretAccessKey, l // SignV4 sign the request before Do(), in accordance with // http://docs.aws.amazon.com/AmazonS3/latest/API/sig-v4-authenticating-requests.html. func SignV4(req http.Request, accessKeyID, secretAccessKey, location string) *http.Request { + // Signature calculation is not needed for anonymous credentials. + if accessKeyID == "" || secretAccessKey == "" { + return &req + } + // Initial time. t := time.Now().UTC() @@ -255,19 +276,19 @@ func SignV4(req http.Request, accessKeyID, secretAccessKey, location string) *ht // Get string to sign from canonical request. stringToSign := getStringToSignV4(t, location, canonicalRequest) - // get hmac signing key. + // Get hmac signing key. signingKey := getSigningKey(secretAccessKey, location, t) - // get credential string. + // Get credential string. credential := getCredential(accessKeyID, location, t) // Get all signed headers. signedHeaders := getSignedHeaders(req) - // calculate signature. + // Calculate signature. signature := getSignature(signingKey, stringToSign) - // if regular request, construct the final authorization header. + // If regular request, construct the final authorization header. parts := []string{ signV4Algorithm + " Credential=" + credential, "SignedHeaders=" + signedHeaders, diff --git a/Godeps/_workspace/src/github.com/minio/minio-go/signature-type.go b/Godeps/_workspace/src/github.com/minio/minio-go/signature-type.go index 8eec3f0eb..cae74cd01 100644 --- a/Godeps/_workspace/src/github.com/minio/minio-go/signature-type.go +++ b/Godeps/_workspace/src/github.com/minio/minio-go/signature-type.go @@ -1,3 +1,19 @@ +/* + * Minio Go Library for Amazon S3 Compatible Cloud Storage (C) 2015 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package minio // SignatureType is type of Authorization requested for a given HTTP request. diff --git a/Godeps/_workspace/src/github.com/minio/minio-go/tempfile.go b/Godeps/_workspace/src/github.com/minio/minio-go/tempfile.go index e9fada3e6..65c7b0da1 100644 --- a/Godeps/_workspace/src/github.com/minio/minio-go/tempfile.go +++ b/Godeps/_workspace/src/github.com/minio/minio-go/tempfile.go @@ -37,7 +37,7 @@ func newTempFile(prefix string) (*tempFile, error) { } return &tempFile{ File: file, - mutex: new(sync.Mutex), + mutex: &sync.Mutex{}, }, nil } diff --git a/Godeps/_workspace/src/github.com/minio/minio-go/utils.go b/Godeps/_workspace/src/github.com/minio/minio-go/utils.go index 2e2532b6c..2d92fc8bc 100644 --- a/Godeps/_workspace/src/github.com/minio/minio-go/utils.go +++ b/Godeps/_workspace/src/github.com/minio/minio-go/utils.go @@ -17,7 +17,10 @@ package minio import ( + "crypto/hmac" + "crypto/sha256" "encoding/hex" + "encoding/xml" "io" "io/ioutil" "net" @@ -29,6 +32,26 @@ import ( "unicode/utf8" ) +// xmlDecoder provide decoded value in xml. +func xmlDecoder(body io.Reader, v interface{}) error { + d := xml.NewDecoder(body) + return d.Decode(v) +} + +// sum256 calculate sha256 sum for an input byte array. +func sum256(data []byte) []byte { + hash := sha256.New() + hash.Write(data) + return hash.Sum(nil) +} + +// sumHMAC calculate hmac between two input byte array. +func sumHMAC(key []byte, data []byte) []byte { + hash := hmac.New(sha256.New, key) + hash.Write(data) + return hash.Sum(nil) +} + // isPartUploaded - true if part is already uploaded. func isPartUploaded(objPart objectPart, objectParts map[int]objectPart) (isUploaded bool) { _, isUploaded = objectParts[objPart.PartNumber] @@ -261,7 +284,6 @@ func isValidObjectPrefix(objectPrefix string) error { // - if input object size is -1 then return maxPartSize. // - if it happens to be that partSize is indeed bigger // than the maximum part size just return maxPartSize. -// func optimalPartSize(objectSize int64) int64 { // if object size is -1 choose part size as 5GiB. if objectSize == -1 {