forked from TrueCloudLab/restic
Update s3 library (again)
This commit is contained in:
parent
181480b68b
commit
0e9236475b
41 changed files with 2793 additions and 1753 deletions
4
Godeps/Godeps.json
generated
4
Godeps/Godeps.json
generated
|
@ -24,8 +24,8 @@
|
|||
},
|
||||
{
|
||||
"ImportPath": "github.com/minio/minio-go",
|
||||
"Comment": "v0.2.5-201-g410319e",
|
||||
"Rev": "410319e0e39a372998f4d9cd2b9da4ff243ae388"
|
||||
"Comment": "v0.2.5-205-g38be406",
|
||||
"Rev": "38be40605dc37d2d7ec06169218365b46ae33e4b"
|
||||
},
|
||||
{
|
||||
"ImportPath": "github.com/pkg/sftp",
|
||||
|
|
2
Godeps/_workspace/src/github.com/minio/minio-go/.travis.yml
generated
vendored
2
Godeps/_workspace/src/github.com/minio/minio-go/.travis.yml
generated
vendored
|
@ -15,7 +15,7 @@ go:
|
|||
|
||||
script:
|
||||
- go vet ./...
|
||||
- go test -test.short -race -v ./...
|
||||
- go test -short -race -v ./...
|
||||
|
||||
notifications:
|
||||
slack:
|
||||
|
|
10
Godeps/_workspace/src/github.com/minio/minio-go/README.md
generated
vendored
10
Godeps/_workspace/src/github.com/minio/minio-go/README.md
generated
vendored
|
@ -67,14 +67,14 @@ func main() {
|
|||
* [RemoveBucket(bucketName) error](examples/s3/removebucket.go)
|
||||
* [GetBucketACL(bucketName) (BucketACL, error)](examples/s3/getbucketacl.go)
|
||||
* [SetBucketACL(bucketName, BucketACL) error)](examples/s3/setbucketacl.go)
|
||||
* [ListBuckets() []BucketStat](examples/s3/listbuckets.go)
|
||||
* [ListObjects(bucketName, objectPrefix, recursive, chan<- struct{}) <-chan ObjectStat](examples/s3/listobjects.go)
|
||||
* [ListIncompleteUploads(bucketName, prefix, recursive, chan<- struct{}) <-chan ObjectMultipartStat](examples/s3/listincompleteuploads.go)
|
||||
* [ListBuckets() []BucketInfo](examples/s3/listbuckets.go)
|
||||
* [ListObjects(bucketName, objectPrefix, recursive, chan<- struct{}) <-chan ObjectInfo](examples/s3/listobjects.go)
|
||||
* [ListIncompleteUploads(bucketName, prefix, recursive, chan<- struct{}) <-chan ObjectMultipartInfo](examples/s3/listincompleteuploads.go)
|
||||
|
||||
### Object Operations.
|
||||
* [PutObject(bucketName, objectName, io.Reader, size, contentType) error](examples/s3/putobject.go)
|
||||
* [GetObject(bucketName, objectName) (io.ReadCloser, ObjectStat, error)](examples/s3/getobject.go)
|
||||
* [StatObject(bucketName, objectName) (ObjectStat, error)](examples/s3/statobject.go)
|
||||
* [GetObject(bucketName, objectName) (io.ReadCloser, ObjectInfo, error)](examples/s3/getobject.go)
|
||||
* [StatObject(bucketName, objectName) (ObjectInfo, error)](examples/s3/statobject.go)
|
||||
* [RemoveObject(bucketName, objectName) error](examples/s3/removeobject.go)
|
||||
* [RemoveIncompleteUpload(bucketName, objectName) <-chan error](examples/s3/removeincompleteupload.go)
|
||||
|
||||
|
|
53
Godeps/_workspace/src/github.com/minio/minio-go/api-definitions.go
generated
vendored
53
Godeps/_workspace/src/github.com/minio/minio-go/api-definitions.go
generated
vendored
|
@ -16,26 +16,27 @@
|
|||
|
||||
package minio
|
||||
|
||||
import (
|
||||
"io"
|
||||
"time"
|
||||
)
|
||||
import "time"
|
||||
|
||||
// BucketStat container for bucket metadata.
|
||||
type BucketStat struct {
|
||||
// BucketInfo container for bucket metadata.
|
||||
type BucketInfo struct {
|
||||
// The name of the bucket.
|
||||
Name string
|
||||
// Date the bucket was created.
|
||||
CreationDate time.Time
|
||||
}
|
||||
|
||||
// ObjectStat container for object metadata.
|
||||
type ObjectStat struct {
|
||||
// ObjectInfo container for object metadata.
|
||||
type ObjectInfo struct {
|
||||
// An ETag is optionally set to md5sum of an object. In case of multipart objects,
|
||||
// ETag is of the form MD5SUM-N where MD5SUM is md5sum of all individual md5sums of
|
||||
// each parts concatenated into one string.
|
||||
ETag string
|
||||
Key string
|
||||
LastModified time.Time
|
||||
Size int64
|
||||
ContentType string
|
||||
|
||||
Key string // Name of the object
|
||||
LastModified time.Time // Date and time the object was last modified.
|
||||
Size int64 // Size in bytes of the object.
|
||||
ContentType string // A standard MIME type describing the format of the object data.
|
||||
|
||||
// Owner name.
|
||||
Owner struct {
|
||||
|
@ -50,18 +51,21 @@ type ObjectStat struct {
|
|||
Err error
|
||||
}
|
||||
|
||||
// ObjectMultipartStat container for multipart object metadata.
|
||||
type ObjectMultipartStat struct {
|
||||
// ObjectMultipartInfo container for multipart object metadata.
|
||||
type ObjectMultipartInfo struct {
|
||||
// Date and time at which the multipart upload was initiated.
|
||||
Initiated time.Time `type:"timestamp" timestampFormat:"iso8601"`
|
||||
|
||||
Initiator initiator
|
||||
Owner owner
|
||||
|
||||
// The type of storage to use for the object. Defaults to 'STANDARD'.
|
||||
StorageClass string
|
||||
|
||||
// Key of the object for which the multipart upload was initiated.
|
||||
Key string
|
||||
|
||||
// Size in bytes of the object.
|
||||
Size int64
|
||||
|
||||
// Upload ID that identifies the multipart upload.
|
||||
|
@ -70,24 +74,3 @@ type ObjectMultipartStat struct {
|
|||
// Error
|
||||
Err error
|
||||
}
|
||||
|
||||
// partData - container for each part.
|
||||
type partData struct {
|
||||
MD5Sum []byte
|
||||
Sha256Sum []byte
|
||||
ReadCloser io.ReadCloser
|
||||
Size int64
|
||||
Number int // partData number.
|
||||
|
||||
// Error
|
||||
Err error
|
||||
}
|
||||
|
||||
// putObjectData - container for each single PUT operation.
|
||||
type putObjectData struct {
|
||||
MD5Sum []byte
|
||||
Sha256Sum []byte
|
||||
ReadCloser io.ReadCloser
|
||||
Size int64
|
||||
ContentType string
|
||||
}
|
||||
|
|
93
Godeps/_workspace/src/github.com/minio/minio-go/api-error-response.go
generated
vendored
93
Godeps/_workspace/src/github.com/minio/minio-go/api-error-response.go
generated
vendored
|
@ -17,7 +17,6 @@
|
|||
package minio
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"encoding/xml"
|
||||
"fmt"
|
||||
"net/http"
|
||||
|
@ -36,7 +35,7 @@ import (
|
|||
</Error>
|
||||
*/
|
||||
|
||||
// ErrorResponse is the type error returned by some API operations.
|
||||
// ErrorResponse - Is the typed error returned by all API operations.
|
||||
type ErrorResponse struct {
|
||||
XMLName xml.Name `xml:"Error" json:"-"`
|
||||
Code string
|
||||
|
@ -46,12 +45,13 @@ type ErrorResponse struct {
|
|||
RequestID string `xml:"RequestId"`
|
||||
HostID string `xml:"HostId"`
|
||||
|
||||
// This is a new undocumented field, set only if available.
|
||||
// Region where the bucket is located. This header is returned
|
||||
// only in HEAD bucket and ListObjects response.
|
||||
AmzBucketRegion string
|
||||
}
|
||||
|
||||
// ToErrorResponse returns parsed ErrorResponse struct, if input is nil or not ErrorResponse return value is nil
|
||||
// this fuction is useful when some one wants to dig deeper into the error structures over the network.
|
||||
// ToErrorResponse - Returns parsed ErrorResponse struct from body and
|
||||
// http headers.
|
||||
//
|
||||
// For example:
|
||||
//
|
||||
|
@ -61,7 +61,6 @@ type ErrorResponse struct {
|
|||
// reader, stat, err := s3.GetObject(...)
|
||||
// if err != nil {
|
||||
// resp := s3.ToErrorResponse(err)
|
||||
// fmt.Println(resp.ToXML())
|
||||
// }
|
||||
// ...
|
||||
func ToErrorResponse(err error) ErrorResponse {
|
||||
|
@ -73,47 +72,32 @@ func ToErrorResponse(err error) ErrorResponse {
|
|||
}
|
||||
}
|
||||
|
||||
// ToXML send raw xml marshalled as string
|
||||
func (e ErrorResponse) ToXML() string {
|
||||
b, err := xml.Marshal(&e)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
return string(b)
|
||||
}
|
||||
|
||||
// ToJSON send raw json marshalled as string
|
||||
func (e ErrorResponse) ToJSON() string {
|
||||
b, err := json.Marshal(&e)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
return string(b)
|
||||
}
|
||||
|
||||
// Error formats HTTP error string
|
||||
// Error - Returns HTTP error string
|
||||
func (e ErrorResponse) Error() string {
|
||||
return e.Message
|
||||
}
|
||||
|
||||
// Common reporting string
|
||||
// Common string for errors to report issue location in unexpected
|
||||
// cases.
|
||||
const (
|
||||
reportIssue = "Please report this issue at https://github.com/minio/minio-go/issues."
|
||||
)
|
||||
|
||||
// HTTPRespToErrorResponse returns a new encoded ErrorResponse structure
|
||||
// HTTPRespToErrorResponse returns a new encoded ErrorResponse
|
||||
// structure as error.
|
||||
func HTTPRespToErrorResponse(resp *http.Response, bucketName, objectName string) error {
|
||||
if resp == nil {
|
||||
msg := "Response is empty. " + reportIssue
|
||||
return ErrInvalidArgument(msg)
|
||||
}
|
||||
var errorResponse ErrorResponse
|
||||
err := xmlDecoder(resp.Body, &errorResponse)
|
||||
var errResp ErrorResponse
|
||||
err := xmlDecoder(resp.Body, &errResp)
|
||||
// Xml decoding failed with no body, fall back to HTTP headers.
|
||||
if err != nil {
|
||||
switch resp.StatusCode {
|
||||
case http.StatusNotFound:
|
||||
if objectName == "" {
|
||||
errorResponse = ErrorResponse{
|
||||
errResp = ErrorResponse{
|
||||
Code: "NoSuchBucket",
|
||||
Message: "The specified bucket does not exist.",
|
||||
BucketName: bucketName,
|
||||
|
@ -122,7 +106,7 @@ func HTTPRespToErrorResponse(resp *http.Response, bucketName, objectName string)
|
|||
AmzBucketRegion: resp.Header.Get("x-amz-bucket-region"),
|
||||
}
|
||||
} else {
|
||||
errorResponse = ErrorResponse{
|
||||
errResp = ErrorResponse{
|
||||
Code: "NoSuchKey",
|
||||
Message: "The specified key does not exist.",
|
||||
BucketName: bucketName,
|
||||
|
@ -133,7 +117,7 @@ func HTTPRespToErrorResponse(resp *http.Response, bucketName, objectName string)
|
|||
}
|
||||
}
|
||||
case http.StatusForbidden:
|
||||
errorResponse = ErrorResponse{
|
||||
errResp = ErrorResponse{
|
||||
Code: "AccessDenied",
|
||||
Message: "Access Denied.",
|
||||
BucketName: bucketName,
|
||||
|
@ -143,7 +127,7 @@ func HTTPRespToErrorResponse(resp *http.Response, bucketName, objectName string)
|
|||
AmzBucketRegion: resp.Header.Get("x-amz-bucket-region"),
|
||||
}
|
||||
case http.StatusConflict:
|
||||
errorResponse = ErrorResponse{
|
||||
errResp = ErrorResponse{
|
||||
Code: "Conflict",
|
||||
Message: "Bucket not empty.",
|
||||
BucketName: bucketName,
|
||||
|
@ -152,7 +136,7 @@ func HTTPRespToErrorResponse(resp *http.Response, bucketName, objectName string)
|
|||
AmzBucketRegion: resp.Header.Get("x-amz-bucket-region"),
|
||||
}
|
||||
default:
|
||||
errorResponse = ErrorResponse{
|
||||
errResp = ErrorResponse{
|
||||
Code: resp.Status,
|
||||
Message: resp.Status,
|
||||
BucketName: bucketName,
|
||||
|
@ -162,10 +146,21 @@ func HTTPRespToErrorResponse(resp *http.Response, bucketName, objectName string)
|
|||
}
|
||||
}
|
||||
}
|
||||
return errorResponse
|
||||
|
||||
// AccessDenied without a signature mismatch code, usually means
|
||||
// that the bucket policy has certain restrictions where some API
|
||||
// operations are not allowed. Handle this case so that top level
|
||||
// callers can interpret this easily and fall back if needed to a
|
||||
// lower functionality call. Read each individual API specific
|
||||
// code for such fallbacks.
|
||||
if errResp.Code == "AccessDenied" && errResp.Message == "Access Denied" {
|
||||
errResp.Code = "NotImplemented"
|
||||
errResp.Message = "Operation is not allowed according to your bucket policy."
|
||||
}
|
||||
return errResp
|
||||
}
|
||||
|
||||
// ErrEntityTooLarge input size is larger than supported maximum.
|
||||
// ErrEntityTooLarge - Input size is larger than supported maximum.
|
||||
func ErrEntityTooLarge(totalSize int64, bucketName, objectName string) error {
|
||||
msg := fmt.Sprintf("Your proposed upload size ‘%d’ exceeds the maximum allowed object size '5GiB' for single PUT operation.", totalSize)
|
||||
return ErrorResponse{
|
||||
|
@ -176,7 +171,19 @@ func ErrEntityTooLarge(totalSize int64, bucketName, objectName string) error {
|
|||
}
|
||||
}
|
||||
|
||||
// ErrUnexpectedShortRead unexpected shorter read of input buffer from target.
|
||||
// ErrEntityTooSmall - Input size is smaller than supported minimum.
|
||||
func ErrEntityTooSmall(totalSize int64, bucketName, objectName string) error {
|
||||
msg := fmt.Sprintf("Your proposed upload size ‘%d’ is below the minimum allowed object size '0B' for single PUT operation.", totalSize)
|
||||
return ErrorResponse{
|
||||
Code: "EntityTooLarge",
|
||||
Message: msg,
|
||||
BucketName: bucketName,
|
||||
Key: objectName,
|
||||
}
|
||||
}
|
||||
|
||||
// ErrUnexpectedShortRead - Unexpected shorter read of input buffer from
|
||||
// target.
|
||||
func ErrUnexpectedShortRead(totalRead, totalSize int64, bucketName, objectName string) error {
|
||||
msg := fmt.Sprintf("Data read ‘%s’ is shorter than the size ‘%s’ of input buffer.",
|
||||
strconv.FormatInt(totalRead, 10), strconv.FormatInt(totalSize, 10))
|
||||
|
@ -188,7 +195,7 @@ func ErrUnexpectedShortRead(totalRead, totalSize int64, bucketName, objectName s
|
|||
}
|
||||
}
|
||||
|
||||
// ErrUnexpectedEOF unexpected end of file reached.
|
||||
// ErrUnexpectedEOF - Unexpected end of file reached.
|
||||
func ErrUnexpectedEOF(totalRead, totalSize int64, bucketName, objectName string) error {
|
||||
msg := fmt.Sprintf("Data read ‘%s’ is not equal to the size ‘%s’ of the input Reader.",
|
||||
strconv.FormatInt(totalRead, 10), strconv.FormatInt(totalSize, 10))
|
||||
|
@ -200,7 +207,7 @@ func ErrUnexpectedEOF(totalRead, totalSize int64, bucketName, objectName string)
|
|||
}
|
||||
}
|
||||
|
||||
// ErrInvalidBucketName - invalid bucket name response.
|
||||
// ErrInvalidBucketName - Invalid bucket name response.
|
||||
func ErrInvalidBucketName(message string) error {
|
||||
return ErrorResponse{
|
||||
Code: "InvalidBucketName",
|
||||
|
@ -209,7 +216,7 @@ func ErrInvalidBucketName(message string) error {
|
|||
}
|
||||
}
|
||||
|
||||
// ErrInvalidObjectName - invalid object name response.
|
||||
// ErrInvalidObjectName - Invalid object name response.
|
||||
func ErrInvalidObjectName(message string) error {
|
||||
return ErrorResponse{
|
||||
Code: "NoSuchKey",
|
||||
|
@ -218,7 +225,7 @@ func ErrInvalidObjectName(message string) error {
|
|||
}
|
||||
}
|
||||
|
||||
// ErrInvalidParts - invalid number of parts.
|
||||
// ErrInvalidParts - Invalid number of parts.
|
||||
func ErrInvalidParts(expectedParts, uploadedParts int) error {
|
||||
msg := fmt.Sprintf("Unexpected number of parts found Want %d, Got %d", expectedParts, uploadedParts)
|
||||
return ErrorResponse{
|
||||
|
@ -228,11 +235,11 @@ func ErrInvalidParts(expectedParts, uploadedParts int) error {
|
|||
}
|
||||
}
|
||||
|
||||
// ErrInvalidObjectPrefix - invalid object prefix response is
|
||||
// ErrInvalidObjectPrefix - Invalid object prefix response is
|
||||
// similar to object name response.
|
||||
var ErrInvalidObjectPrefix = ErrInvalidObjectName
|
||||
|
||||
// ErrInvalidArgument - invalid argument response.
|
||||
// ErrInvalidArgument - Invalid argument response.
|
||||
func ErrInvalidArgument(message string) error {
|
||||
return ErrorResponse{
|
||||
Code: "InvalidArgument",
|
||||
|
|
|
@ -22,7 +22,7 @@ import (
|
|||
"path/filepath"
|
||||
)
|
||||
|
||||
// FGetObject - get object to a file.
|
||||
// FGetObject - download contents of an object to a local file.
|
||||
func (c Client) FGetObject(bucketName, objectName, filePath string) error {
|
||||
// Input validation.
|
||||
if err := isValidBucketName(bucketName); err != nil {
|
345
Godeps/_workspace/src/github.com/minio/minio-go/api-get.go
generated
vendored
345
Godeps/_workspace/src/github.com/minio/minio-go/api-get.go
generated
vendored
|
@ -28,15 +28,18 @@ import (
|
|||
"time"
|
||||
)
|
||||
|
||||
// GetBucketACL get the permissions on an existing bucket.
|
||||
// GetBucketACL - Get the permissions on an existing bucket.
|
||||
//
|
||||
// Returned values are:
|
||||
//
|
||||
// private - owner gets full access.
|
||||
// public-read - owner gets full access, others get read access.
|
||||
// public-read-write - owner gets full access, others get full access too.
|
||||
// authenticated-read - owner gets full access, authenticated users get read access.
|
||||
// private - Owner gets full access.
|
||||
// public-read - Owner gets full access, others get read access.
|
||||
// public-read-write - Owner gets full access, others get full access
|
||||
// too.
|
||||
// authenticated-read - Owner gets full access, authenticated users
|
||||
// get read access.
|
||||
func (c Client) GetBucketACL(bucketName string) (BucketACL, error) {
|
||||
// Input validation.
|
||||
if err := isValidBucketName(bucketName); err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
@ -73,9 +76,10 @@ func (c Client) GetBucketACL(bucketName string) (BucketACL, error) {
|
|||
return "", err
|
||||
}
|
||||
|
||||
// We need to avoid following de-serialization check for Google Cloud Storage.
|
||||
// On Google Cloud Storage "private" canned ACL's policy do not have grant list.
|
||||
// Treat it as a valid case, check for all other vendors.
|
||||
// We need to avoid following de-serialization check for Google
|
||||
// Cloud Storage. On Google Cloud Storage "private" canned ACL's
|
||||
// policy do not have grant list. Treat it as a valid case, check
|
||||
// for all other vendors.
|
||||
if !isGoogleEndpoint(c.endpointURL) {
|
||||
if policy.AccessControlList.Grant == nil {
|
||||
errorResponse := ErrorResponse{
|
||||
|
@ -90,8 +94,8 @@ func (c Client) GetBucketACL(bucketName string) (BucketACL, error) {
|
|||
}
|
||||
}
|
||||
|
||||
// boolean cues to indentify right canned acls.
|
||||
var publicRead, publicWrite bool
|
||||
// Boolean cues to indentify right canned acls.
|
||||
var publicRead, publicWrite, authenticatedRead bool
|
||||
|
||||
// Handle grants.
|
||||
grants := policy.AccessControlList.Grant
|
||||
|
@ -100,7 +104,8 @@ func (c Client) GetBucketACL(bucketName string) (BucketACL, error) {
|
|||
continue
|
||||
}
|
||||
if g.Grantee.URI == "http://acs.amazonaws.com/groups/global/AuthenticatedUsers" && g.Permission == "READ" {
|
||||
return BucketACL("authenticated-read"), nil
|
||||
authenticatedRead = true
|
||||
break
|
||||
} else if g.Grantee.URI == "http://acs.amazonaws.com/groups/global/AllUsers" && g.Permission == "WRITE" {
|
||||
publicWrite = true
|
||||
} else if g.Grantee.URI == "http://acs.amazonaws.com/groups/global/AllUsers" && g.Permission == "READ" {
|
||||
|
@ -108,15 +113,19 @@ func (c Client) GetBucketACL(bucketName string) (BucketACL, error) {
|
|||
}
|
||||
}
|
||||
|
||||
// public write and not enabled. return.
|
||||
// Verify if acl is authenticated read.
|
||||
if authenticatedRead {
|
||||
return BucketACL("authenticated-read"), nil
|
||||
}
|
||||
// Verify if acl is private.
|
||||
if !publicWrite && !publicRead {
|
||||
return BucketACL("private"), nil
|
||||
}
|
||||
// public write not enabled but public read is. return.
|
||||
// Verify if acl is public-read.
|
||||
if !publicWrite && publicRead {
|
||||
return BucketACL("public-read"), nil
|
||||
}
|
||||
// public read and public write are enabled return.
|
||||
// Verify if acl is public-read-write.
|
||||
if publicRead && publicWrite {
|
||||
return BucketACL("public-read-write"), nil
|
||||
}
|
||||
|
@ -129,47 +138,30 @@ func (c Client) GetBucketACL(bucketName string) (BucketACL, error) {
|
|||
}
|
||||
}
|
||||
|
||||
// GetObject gets object content from specified bucket.
|
||||
// You may also look at GetPartialObject.
|
||||
func (c Client) GetObject(bucketName, objectName string) (io.ReadCloser, ObjectStat, error) {
|
||||
// GetObject - returns an seekable, readable object.
|
||||
func (c Client) GetObject(bucketName, objectName string) (*Object, error) {
|
||||
// Input validation.
|
||||
if err := isValidBucketName(bucketName); err != nil {
|
||||
return nil, ObjectStat{}, err
|
||||
return nil, err
|
||||
}
|
||||
if err := isValidObjectName(objectName); err != nil {
|
||||
return nil, ObjectStat{}, err
|
||||
return nil, err
|
||||
}
|
||||
// get the whole object as a stream, no seek or resume supported for this.
|
||||
return c.getObject(bucketName, objectName, 0, 0)
|
||||
}
|
||||
|
||||
// ReadAtCloser readat closer interface.
|
||||
type ReadAtCloser interface {
|
||||
io.ReaderAt
|
||||
io.Closer
|
||||
}
|
||||
|
||||
// GetObjectPartial returns a io.ReadAt for reading sparse entries.
|
||||
func (c Client) GetObjectPartial(bucketName, objectName string) (ReadAtCloser, ObjectStat, error) {
|
||||
if err := isValidBucketName(bucketName); err != nil {
|
||||
return nil, ObjectStat{}, err
|
||||
}
|
||||
if err := isValidObjectName(objectName); err != nil {
|
||||
return nil, ObjectStat{}, err
|
||||
}
|
||||
// Send an explicit stat to get the actual object size.
|
||||
objectStat, err := c.StatObject(bucketName, objectName)
|
||||
// Send an explicit info to get the actual object size.
|
||||
objectInfo, err := c.StatObject(bucketName, objectName)
|
||||
if err != nil {
|
||||
return nil, ObjectStat{}, err
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Create request channel.
|
||||
reqCh := make(chan readAtRequest)
|
||||
reqCh := make(chan readRequest)
|
||||
// Create response channel.
|
||||
resCh := make(chan readAtResponse)
|
||||
resCh := make(chan readResponse)
|
||||
// Create done channel.
|
||||
doneCh := make(chan struct{})
|
||||
|
||||
// This routine feeds partial object data as and when the caller reads.
|
||||
// This routine feeds partial object data as and when the caller
|
||||
// reads.
|
||||
go func() {
|
||||
defer close(reqCh)
|
||||
defer close(resCh)
|
||||
|
@ -185,21 +177,21 @@ func (c Client) GetObjectPartial(bucketName, objectName string) (ReadAtCloser, O
|
|||
// Get shortest length.
|
||||
// NOTE: Last remaining bytes are usually smaller than
|
||||
// req.Buffer size. Use that as the final length.
|
||||
length := math.Min(float64(len(req.Buffer)), float64(objectStat.Size-req.Offset))
|
||||
length := math.Min(float64(len(req.Buffer)), float64(objectInfo.Size-req.Offset))
|
||||
httpReader, _, err := c.getObject(bucketName, objectName, req.Offset, int64(length))
|
||||
if err != nil {
|
||||
resCh <- readAtResponse{
|
||||
resCh <- readResponse{
|
||||
Error: err,
|
||||
}
|
||||
return
|
||||
}
|
||||
size, err := io.ReadFull(httpReader, req.Buffer)
|
||||
if err == io.ErrUnexpectedEOF {
|
||||
// If an EOF happens after reading some but not all the bytes
|
||||
// ReadFull returns ErrUnexpectedEOF
|
||||
// If an EOF happens after reading some but not
|
||||
// all the bytes ReadFull returns ErrUnexpectedEOF
|
||||
err = io.EOF
|
||||
}
|
||||
resCh <- readAtResponse{
|
||||
resCh <- readResponse{
|
||||
Size: int(size),
|
||||
Error: err,
|
||||
}
|
||||
|
@ -207,78 +199,148 @@ func (c Client) GetObjectPartial(bucketName, objectName string) (ReadAtCloser, O
|
|||
}
|
||||
}()
|
||||
// Return the readerAt backed by routine.
|
||||
return newObjectReadAtCloser(reqCh, resCh, doneCh, objectStat.Size), objectStat, nil
|
||||
return newObject(reqCh, resCh, doneCh, objectInfo), nil
|
||||
}
|
||||
|
||||
// response message container to reply back for the request.
|
||||
type readAtResponse struct {
|
||||
// Read response message container to reply back for the request.
|
||||
type readResponse struct {
|
||||
Size int
|
||||
Error error
|
||||
}
|
||||
|
||||
// request message container to communicate with internal go-routine.
|
||||
type readAtRequest struct {
|
||||
// Read request message container to communicate with internal
|
||||
// go-routine.
|
||||
type readRequest struct {
|
||||
Buffer []byte
|
||||
Offset int64 // readAt offset.
|
||||
}
|
||||
|
||||
// objectReadAtCloser container for io.ReadAtCloser.
|
||||
type objectReadAtCloser struct {
|
||||
// mutex.
|
||||
// Object represents an open object. It implements Read, ReadAt,
|
||||
// Seeker, Close for a HTTP stream.
|
||||
type Object struct {
|
||||
// Mutex.
|
||||
mutex *sync.Mutex
|
||||
|
||||
// User allocated and defined.
|
||||
reqCh chan<- readAtRequest
|
||||
resCh <-chan readAtResponse
|
||||
reqCh chan<- readRequest
|
||||
resCh <-chan readResponse
|
||||
doneCh chan<- struct{}
|
||||
objectSize int64
|
||||
currOffset int64
|
||||
objectInfo ObjectInfo
|
||||
|
||||
// Previous error saved for future calls.
|
||||
prevErr error
|
||||
}
|
||||
|
||||
// newObjectReadAtCloser implements a io.ReadSeeker for a HTTP stream.
|
||||
func newObjectReadAtCloser(reqCh chan<- readAtRequest, resCh <-chan readAtResponse, doneCh chan<- struct{}, objectSize int64) *objectReadAtCloser {
|
||||
return &objectReadAtCloser{
|
||||
mutex: new(sync.Mutex),
|
||||
reqCh: reqCh,
|
||||
resCh: resCh,
|
||||
doneCh: doneCh,
|
||||
objectSize: objectSize,
|
||||
}
|
||||
// Read reads up to len(p) bytes into p. It returns the number of
|
||||
// bytes read (0 <= n <= len(p)) and any error encountered. Returns
|
||||
// io.EOF upon end of file.
|
||||
func (o *Object) Read(b []byte) (n int, err error) {
|
||||
if o == nil {
|
||||
return 0, ErrInvalidArgument("Object is nil")
|
||||
}
|
||||
|
||||
// ReadAt reads len(b) bytes from the File starting at byte offset off.
|
||||
// It returns the number of bytes read and the error, if any.
|
||||
// ReadAt always returns a non-nil error when n < len(b).
|
||||
// At end of file, that error is io.EOF.
|
||||
func (r *objectReadAtCloser) ReadAt(b []byte, offset int64) (int, error) {
|
||||
// Locking.
|
||||
r.mutex.Lock()
|
||||
defer r.mutex.Unlock()
|
||||
o.mutex.Lock()
|
||||
defer o.mutex.Unlock()
|
||||
|
||||
// if offset is negative and offset is greater than or equal to object size we return EOF.
|
||||
if offset < 0 || offset >= r.objectSize {
|
||||
// If current offset has reached Size limit, return EOF.
|
||||
if o.currOffset >= o.objectInfo.Size {
|
||||
return 0, io.EOF
|
||||
}
|
||||
|
||||
// Previous prevErr is which was saved in previous operation.
|
||||
if o.prevErr != nil {
|
||||
return 0, o.prevErr
|
||||
}
|
||||
|
||||
// Send current information over control channel to indicate we
|
||||
// are ready.
|
||||
reqMsg := readRequest{}
|
||||
|
||||
// Send the offset and pointer to the buffer over the channel.
|
||||
reqMsg.Buffer = b
|
||||
reqMsg.Offset = o.currOffset
|
||||
|
||||
// Send read request over the control channel.
|
||||
o.reqCh <- reqMsg
|
||||
|
||||
// Get data over the response channel.
|
||||
dataMsg := <-o.resCh
|
||||
|
||||
// Bytes read.
|
||||
bytesRead := int64(dataMsg.Size)
|
||||
|
||||
// Update current offset.
|
||||
o.currOffset += bytesRead
|
||||
|
||||
if dataMsg.Error == nil {
|
||||
// If currOffset read is equal to objectSize
|
||||
// We have reached end of file, we return io.EOF.
|
||||
if o.currOffset >= o.objectInfo.Size {
|
||||
return dataMsg.Size, io.EOF
|
||||
}
|
||||
return dataMsg.Size, nil
|
||||
}
|
||||
|
||||
// Save any error.
|
||||
o.prevErr = dataMsg.Error
|
||||
return dataMsg.Size, dataMsg.Error
|
||||
}
|
||||
|
||||
// Stat returns the ObjectInfo structure describing object.
|
||||
func (o *Object) Stat() (ObjectInfo, error) {
|
||||
if o == nil {
|
||||
return ObjectInfo{}, ErrInvalidArgument("Object is nil")
|
||||
}
|
||||
// Locking.
|
||||
o.mutex.Lock()
|
||||
defer o.mutex.Unlock()
|
||||
|
||||
if o.prevErr != nil {
|
||||
return ObjectInfo{}, o.prevErr
|
||||
}
|
||||
|
||||
return o.objectInfo, nil
|
||||
}
|
||||
|
||||
// ReadAt reads len(b) bytes from the File starting at byte offset
|
||||
// off. It returns the number of bytes read and the error, if any.
|
||||
// ReadAt always returns a non-nil error when n < len(b). At end of
|
||||
// file, that error is io.EOF.
|
||||
func (o *Object) ReadAt(b []byte, offset int64) (n int, err error) {
|
||||
if o == nil {
|
||||
return 0, ErrInvalidArgument("Object is nil")
|
||||
}
|
||||
|
||||
// Locking.
|
||||
o.mutex.Lock()
|
||||
defer o.mutex.Unlock()
|
||||
|
||||
// If offset is negative and offset is greater than or equal to
|
||||
// object size we return EOF.
|
||||
if offset < 0 || offset >= o.objectInfo.Size {
|
||||
return 0, io.EOF
|
||||
}
|
||||
|
||||
// prevErr is which was saved in previous operation.
|
||||
if r.prevErr != nil {
|
||||
return 0, r.prevErr
|
||||
if o.prevErr != nil {
|
||||
return 0, o.prevErr
|
||||
}
|
||||
|
||||
// Send current information over control channel to indicate we are ready.
|
||||
reqMsg := readAtRequest{}
|
||||
// Send current information over control channel to indicate we
|
||||
// are ready.
|
||||
reqMsg := readRequest{}
|
||||
|
||||
// Send the current offset and bytes requested.
|
||||
// Send the offset and pointer to the buffer over the channel.
|
||||
reqMsg.Buffer = b
|
||||
reqMsg.Offset = offset
|
||||
|
||||
// Send read request over the control channel.
|
||||
r.reqCh <- reqMsg
|
||||
o.reqCh <- reqMsg
|
||||
|
||||
// Get data over the response channel.
|
||||
dataMsg := <-r.resCh
|
||||
dataMsg := <-o.resCh
|
||||
|
||||
// Bytes read.
|
||||
bytesRead := int64(dataMsg.Size)
|
||||
|
@ -286,38 +348,109 @@ func (r *objectReadAtCloser) ReadAt(b []byte, offset int64) (int, error) {
|
|||
if dataMsg.Error == nil {
|
||||
// If offset+bytes read is equal to objectSize
|
||||
// we have reached end of file, we return io.EOF.
|
||||
if offset+bytesRead == r.objectSize {
|
||||
if offset+bytesRead == o.objectInfo.Size {
|
||||
return dataMsg.Size, io.EOF
|
||||
}
|
||||
return dataMsg.Size, nil
|
||||
}
|
||||
|
||||
// Save any error.
|
||||
r.prevErr = dataMsg.Error
|
||||
o.prevErr = dataMsg.Error
|
||||
return dataMsg.Size, dataMsg.Error
|
||||
}
|
||||
|
||||
// Closer is the interface that wraps the basic Close method.
|
||||
// Seek sets the offset for the next Read or Write to offset,
|
||||
// interpreted according to whence: 0 means relative to the
|
||||
// origin of the file, 1 means relative to the current offset,
|
||||
// and 2 means relative to the end.
|
||||
// Seek returns the new offset and an error, if any.
|
||||
//
|
||||
// The behavior of Close after the first call returns error for
|
||||
// subsequent Close() calls.
|
||||
func (r *objectReadAtCloser) Close() (err error) {
|
||||
// Seeking to a negative offset is an error. Seeking to any positive
|
||||
// offset is legal, subsequent io operations succeed until the
|
||||
// underlying object is not closed.
|
||||
func (o *Object) Seek(offset int64, whence int) (n int64, err error) {
|
||||
if o == nil {
|
||||
return 0, ErrInvalidArgument("Object is nil")
|
||||
}
|
||||
|
||||
// Locking.
|
||||
r.mutex.Lock()
|
||||
defer r.mutex.Unlock()
|
||||
o.mutex.Lock()
|
||||
defer o.mutex.Unlock()
|
||||
|
||||
if o.prevErr != nil {
|
||||
// At EOF seeking is legal, for any other errors we return.
|
||||
if o.prevErr != io.EOF {
|
||||
return 0, o.prevErr
|
||||
}
|
||||
}
|
||||
|
||||
// Negative offset is valid for whence of '2'.
|
||||
if offset < 0 && whence != 2 {
|
||||
return 0, ErrInvalidArgument(fmt.Sprintf("Object: negative position not allowed for %d.", whence))
|
||||
}
|
||||
switch whence {
|
||||
default:
|
||||
return 0, ErrInvalidArgument(fmt.Sprintf("Object: invalid whence %d", whence))
|
||||
case 0:
|
||||
if offset > o.objectInfo.Size {
|
||||
return 0, io.EOF
|
||||
}
|
||||
o.currOffset = offset
|
||||
case 1:
|
||||
if o.currOffset+offset > o.objectInfo.Size {
|
||||
return 0, io.EOF
|
||||
}
|
||||
o.currOffset += offset
|
||||
case 2:
|
||||
// Seeking to positive offset is valid for whence '2', but
|
||||
// since we are backing a Reader we have reached 'EOF' if
|
||||
// offset is positive.
|
||||
if offset > 0 {
|
||||
return 0, io.EOF
|
||||
}
|
||||
// Seeking to negative position not allowed for whence.
|
||||
if o.objectInfo.Size+offset < 0 {
|
||||
return 0, ErrInvalidArgument(fmt.Sprintf("Object: Seeking at negative offset not allowed for %d", whence))
|
||||
}
|
||||
o.currOffset += offset
|
||||
}
|
||||
// Return the effective offset.
|
||||
return o.currOffset, nil
|
||||
}
|
||||
|
||||
// Close - The behavior of Close after the first call returns error
|
||||
// for subsequent Close() calls.
|
||||
func (o *Object) Close() (err error) {
|
||||
if o == nil {
|
||||
return ErrInvalidArgument("Object is nil")
|
||||
}
|
||||
// Locking.
|
||||
o.mutex.Lock()
|
||||
defer o.mutex.Unlock()
|
||||
|
||||
// prevErr is which was saved in previous operation.
|
||||
if r.prevErr != nil {
|
||||
return r.prevErr
|
||||
if o.prevErr != nil {
|
||||
return o.prevErr
|
||||
}
|
||||
|
||||
// Close successfully.
|
||||
close(r.doneCh)
|
||||
close(o.doneCh)
|
||||
|
||||
// Save this for any subsequent frivolous reads.
|
||||
errMsg := "objectReadAtCloser: is already closed. Bad file descriptor."
|
||||
r.prevErr = errors.New(errMsg)
|
||||
return
|
||||
errMsg := "Object: Is already closed. Bad file descriptor."
|
||||
o.prevErr = errors.New(errMsg)
|
||||
return nil
|
||||
}
|
||||
|
||||
// newObject instantiates a new *minio.Object*
|
||||
func newObject(reqCh chan<- readRequest, resCh <-chan readResponse, doneCh chan<- struct{}, objectInfo ObjectInfo) *Object {
|
||||
return &Object{
|
||||
mutex: &sync.Mutex{},
|
||||
reqCh: reqCh,
|
||||
resCh: resCh,
|
||||
doneCh: doneCh,
|
||||
objectInfo: objectInfo,
|
||||
}
|
||||
}
|
||||
|
||||
// getObject - retrieve object from Object Storage.
|
||||
|
@ -327,13 +460,13 @@ func (r *objectReadAtCloser) Close() (err error) {
|
|||
//
|
||||
// For more information about the HTTP Range header.
|
||||
// go to http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.35.
|
||||
func (c Client) getObject(bucketName, objectName string, offset, length int64) (io.ReadCloser, ObjectStat, error) {
|
||||
func (c Client) getObject(bucketName, objectName string, offset, length int64) (io.ReadCloser, ObjectInfo, error) {
|
||||
// Validate input arguments.
|
||||
if err := isValidBucketName(bucketName); err != nil {
|
||||
return nil, ObjectStat{}, err
|
||||
return nil, ObjectInfo{}, err
|
||||
}
|
||||
if err := isValidObjectName(objectName); err != nil {
|
||||
return nil, ObjectStat{}, err
|
||||
return nil, ObjectInfo{}, err
|
||||
}
|
||||
|
||||
customHeader := make(http.Header)
|
||||
|
@ -353,16 +486,16 @@ func (c Client) getObject(bucketName, objectName string, offset, length int64) (
|
|||
customHeader: customHeader,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, ObjectStat{}, err
|
||||
return nil, ObjectInfo{}, err
|
||||
}
|
||||
// Execute the request.
|
||||
resp, err := c.do(req)
|
||||
if err != nil {
|
||||
return nil, ObjectStat{}, err
|
||||
return nil, ObjectInfo{}, err
|
||||
}
|
||||
if resp != nil {
|
||||
if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusPartialContent {
|
||||
return nil, ObjectStat{}, HTTPRespToErrorResponse(resp, bucketName, objectName)
|
||||
return nil, ObjectInfo{}, HTTPRespToErrorResponse(resp, bucketName, objectName)
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -374,7 +507,7 @@ func (c Client) getObject(bucketName, objectName string, offset, length int64) (
|
|||
date, err := time.Parse(http.TimeFormat, resp.Header.Get("Last-Modified"))
|
||||
if err != nil {
|
||||
msg := "Last-Modified time format not recognized. " + reportIssue
|
||||
return nil, ObjectStat{}, ErrorResponse{
|
||||
return nil, ObjectInfo{}, ErrorResponse{
|
||||
Code: "InternalError",
|
||||
Message: msg,
|
||||
RequestID: resp.Header.Get("x-amz-request-id"),
|
||||
|
@ -387,7 +520,7 @@ func (c Client) getObject(bucketName, objectName string, offset, length int64) (
|
|||
if contentType == "" {
|
||||
contentType = "application/octet-stream"
|
||||
}
|
||||
var objectStat ObjectStat
|
||||
var objectStat ObjectInfo
|
||||
objectStat.ETag = md5sum
|
||||
objectStat.Key = objectName
|
||||
objectStat.Size = resp.ContentLength
|
||||
|
|
95
Godeps/_workspace/src/github.com/minio/minio-go/api-list.go
generated
vendored
95
Godeps/_workspace/src/github.com/minio/minio-go/api-list.go
generated
vendored
|
@ -33,7 +33,7 @@ import (
|
|||
// fmt.Println(message)
|
||||
// }
|
||||
//
|
||||
func (c Client) ListBuckets() ([]BucketStat, error) {
|
||||
func (c Client) ListBuckets() ([]BucketInfo, error) {
|
||||
// Instantiate a new request.
|
||||
req, err := c.newRequest("GET", requestMetadata{})
|
||||
if err != nil {
|
||||
|
@ -64,19 +64,25 @@ func (c Client) ListBuckets() ([]BucketStat, error) {
|
|||
// the specified bucket. If recursion is enabled it would list
|
||||
// all subdirectories and all its contents.
|
||||
//
|
||||
// Your input paramters are just bucketName, objectPrefix and recursive. If you
|
||||
// enable recursive as 'true' this function will return back all the
|
||||
// objects in a given bucket name and object prefix.
|
||||
// Your input paramters are just bucketName, objectPrefix, recursive
|
||||
// and a done channel for pro-actively closing the internal go
|
||||
// routine. If you enable recursive as 'true' this function will
|
||||
// return back all the objects in a given bucket name and object
|
||||
// prefix.
|
||||
//
|
||||
// api := client.New(....)
|
||||
// // Create a done channel.
|
||||
// doneCh := make(chan struct{})
|
||||
// defer close(doneCh)
|
||||
// // Recurively list all objects in 'mytestbucket'
|
||||
// recursive := true
|
||||
// for message := range api.ListObjects("mytestbucket", "starthere", recursive) {
|
||||
// for message := range api.ListObjects("mytestbucket", "starthere", recursive, doneCh) {
|
||||
// fmt.Println(message)
|
||||
// }
|
||||
//
|
||||
func (c Client) ListObjects(bucketName, objectPrefix string, recursive bool, doneCh <-chan struct{}) <-chan ObjectStat {
|
||||
func (c Client) ListObjects(bucketName, objectPrefix string, recursive bool, doneCh <-chan struct{}) <-chan ObjectInfo {
|
||||
// Allocate new list objects channel.
|
||||
objectStatCh := make(chan ObjectStat, 1000)
|
||||
objectStatCh := make(chan ObjectInfo, 1000)
|
||||
// Default listing is delimited at "/"
|
||||
delimiter := "/"
|
||||
if recursive {
|
||||
|
@ -86,7 +92,7 @@ func (c Client) ListObjects(bucketName, objectPrefix string, recursive bool, don
|
|||
// Validate bucket name.
|
||||
if err := isValidBucketName(bucketName); err != nil {
|
||||
defer close(objectStatCh)
|
||||
objectStatCh <- ObjectStat{
|
||||
objectStatCh <- ObjectInfo{
|
||||
Err: err,
|
||||
}
|
||||
return objectStatCh
|
||||
|
@ -94,14 +100,14 @@ func (c Client) ListObjects(bucketName, objectPrefix string, recursive bool, don
|
|||
// Validate incoming object prefix.
|
||||
if err := isValidObjectPrefix(objectPrefix); err != nil {
|
||||
defer close(objectStatCh)
|
||||
objectStatCh <- ObjectStat{
|
||||
objectStatCh <- ObjectInfo{
|
||||
Err: err,
|
||||
}
|
||||
return objectStatCh
|
||||
}
|
||||
|
||||
// Initiate list objects goroutine here.
|
||||
go func(objectStatCh chan<- ObjectStat) {
|
||||
go func(objectStatCh chan<- ObjectInfo) {
|
||||
defer close(objectStatCh)
|
||||
// Save marker for next request.
|
||||
var marker string
|
||||
|
@ -109,7 +115,7 @@ func (c Client) ListObjects(bucketName, objectPrefix string, recursive bool, don
|
|||
// Get list of objects a maximum of 1000 per request.
|
||||
result, err := c.listObjectsQuery(bucketName, objectPrefix, marker, delimiter, 1000)
|
||||
if err != nil {
|
||||
objectStatCh <- ObjectStat{
|
||||
objectStatCh <- ObjectInfo{
|
||||
Err: err,
|
||||
}
|
||||
return
|
||||
|
@ -131,7 +137,7 @@ func (c Client) ListObjects(bucketName, objectPrefix string, recursive bool, don
|
|||
// Send all common prefixes if any.
|
||||
// NOTE: prefixes are only present if the request is delimited.
|
||||
for _, obj := range result.CommonPrefixes {
|
||||
object := ObjectStat{}
|
||||
object := ObjectInfo{}
|
||||
object.Key = obj.Prefix
|
||||
object.Size = 0
|
||||
select {
|
||||
|
@ -181,11 +187,22 @@ func (c Client) listObjectsQuery(bucketName, objectPrefix, objectMarker, delimit
|
|||
// using them in http request.
|
||||
urlValues := make(url.Values)
|
||||
// Set object prefix.
|
||||
if objectPrefix != "" {
|
||||
urlValues.Set("prefix", urlEncodePath(objectPrefix))
|
||||
}
|
||||
// Set object marker.
|
||||
if objectMarker != "" {
|
||||
urlValues.Set("marker", urlEncodePath(objectMarker))
|
||||
}
|
||||
// Set delimiter.
|
||||
if delimiter != "" {
|
||||
urlValues.Set("delimiter", delimiter)
|
||||
}
|
||||
|
||||
// maxkeys should default to 1000 or less.
|
||||
if maxkeys == 0 || maxkeys > 1000 {
|
||||
maxkeys = 1000
|
||||
}
|
||||
// Set max keys.
|
||||
urlValues.Set("max-keys", fmt.Sprintf("%d", maxkeys))
|
||||
|
||||
|
@ -223,26 +240,31 @@ func (c Client) listObjectsQuery(bucketName, objectPrefix, objectMarker, delimit
|
|||
// objectPrefix from the specified bucket. If recursion is enabled
|
||||
// it would list all subdirectories and all its contents.
|
||||
//
|
||||
// Your input paramters are just bucketName, objectPrefix and recursive.
|
||||
// Your input paramters are just bucketName, objectPrefix, recursive
|
||||
// and a done channel to proactively close the internal go routine.
|
||||
// If you enable recursive as 'true' this function will return back all
|
||||
// the multipart objects in a given bucket name.
|
||||
//
|
||||
// api := client.New(....)
|
||||
// // Create a done channel.
|
||||
// doneCh := make(chan struct{})
|
||||
// defer close(doneCh)
|
||||
// // Recurively list all objects in 'mytestbucket'
|
||||
// recursive := true
|
||||
// for message := range api.ListIncompleteUploads("mytestbucket", "starthere", recursive) {
|
||||
// fmt.Println(message)
|
||||
// }
|
||||
//
|
||||
func (c Client) ListIncompleteUploads(bucketName, objectPrefix string, recursive bool, doneCh <-chan struct{}) <-chan ObjectMultipartStat {
|
||||
func (c Client) ListIncompleteUploads(bucketName, objectPrefix string, recursive bool, doneCh <-chan struct{}) <-chan ObjectMultipartInfo {
|
||||
// Turn on size aggregation of individual parts.
|
||||
isAggregateSize := true
|
||||
return c.listIncompleteUploads(bucketName, objectPrefix, recursive, isAggregateSize, doneCh)
|
||||
}
|
||||
|
||||
// listIncompleteUploads lists all incomplete uploads.
|
||||
func (c Client) listIncompleteUploads(bucketName, objectPrefix string, recursive, aggregateSize bool, doneCh <-chan struct{}) <-chan ObjectMultipartStat {
|
||||
func (c Client) listIncompleteUploads(bucketName, objectPrefix string, recursive, aggregateSize bool, doneCh <-chan struct{}) <-chan ObjectMultipartInfo {
|
||||
// Allocate channel for multipart uploads.
|
||||
objectMultipartStatCh := make(chan ObjectMultipartStat, 1000)
|
||||
objectMultipartStatCh := make(chan ObjectMultipartInfo, 1000)
|
||||
// Delimiter is set to "/" by default.
|
||||
delimiter := "/"
|
||||
if recursive {
|
||||
|
@ -252,7 +274,7 @@ func (c Client) listIncompleteUploads(bucketName, objectPrefix string, recursive
|
|||
// Validate bucket name.
|
||||
if err := isValidBucketName(bucketName); err != nil {
|
||||
defer close(objectMultipartStatCh)
|
||||
objectMultipartStatCh <- ObjectMultipartStat{
|
||||
objectMultipartStatCh <- ObjectMultipartInfo{
|
||||
Err: err,
|
||||
}
|
||||
return objectMultipartStatCh
|
||||
|
@ -260,12 +282,12 @@ func (c Client) listIncompleteUploads(bucketName, objectPrefix string, recursive
|
|||
// Validate incoming object prefix.
|
||||
if err := isValidObjectPrefix(objectPrefix); err != nil {
|
||||
defer close(objectMultipartStatCh)
|
||||
objectMultipartStatCh <- ObjectMultipartStat{
|
||||
objectMultipartStatCh <- ObjectMultipartInfo{
|
||||
Err: err,
|
||||
}
|
||||
return objectMultipartStatCh
|
||||
}
|
||||
go func(objectMultipartStatCh chan<- ObjectMultipartStat) {
|
||||
go func(objectMultipartStatCh chan<- ObjectMultipartInfo) {
|
||||
defer close(objectMultipartStatCh)
|
||||
// object and upload ID marker for future requests.
|
||||
var objectMarker string
|
||||
|
@ -274,7 +296,7 @@ func (c Client) listIncompleteUploads(bucketName, objectPrefix string, recursive
|
|||
// list all multipart uploads.
|
||||
result, err := c.listMultipartUploadsQuery(bucketName, objectMarker, uploadIDMarker, objectPrefix, delimiter, 1000)
|
||||
if err != nil {
|
||||
objectMultipartStatCh <- ObjectMultipartStat{
|
||||
objectMultipartStatCh <- ObjectMultipartInfo{
|
||||
Err: err,
|
||||
}
|
||||
return
|
||||
|
@ -289,7 +311,7 @@ func (c Client) listIncompleteUploads(bucketName, objectPrefix string, recursive
|
|||
// Get total multipart size.
|
||||
obj.Size, err = c.getTotalMultipartSize(bucketName, obj.Key, obj.UploadID)
|
||||
if err != nil {
|
||||
objectMultipartStatCh <- ObjectMultipartStat{
|
||||
objectMultipartStatCh <- ObjectMultipartInfo{
|
||||
Err: err,
|
||||
}
|
||||
}
|
||||
|
@ -305,7 +327,7 @@ func (c Client) listIncompleteUploads(bucketName, objectPrefix string, recursive
|
|||
// Send all common prefixes if any.
|
||||
// NOTE: prefixes are only present if the request is delimited.
|
||||
for _, obj := range result.CommonPrefixes {
|
||||
object := ObjectMultipartStat{}
|
||||
object := ObjectMultipartInfo{}
|
||||
object.Key = obj.Prefix
|
||||
object.Size = 0
|
||||
select {
|
||||
|
@ -343,13 +365,26 @@ func (c Client) listMultipartUploadsQuery(bucketName, keyMarker, uploadIDMarker,
|
|||
// Set uploads.
|
||||
urlValues.Set("uploads", "")
|
||||
// Set object key marker.
|
||||
if keyMarker != "" {
|
||||
urlValues.Set("key-marker", urlEncodePath(keyMarker))
|
||||
}
|
||||
// Set upload id marker.
|
||||
if uploadIDMarker != "" {
|
||||
urlValues.Set("upload-id-marker", uploadIDMarker)
|
||||
}
|
||||
// Set prefix marker.
|
||||
if prefix != "" {
|
||||
urlValues.Set("prefix", urlEncodePath(prefix))
|
||||
}
|
||||
// Set delimiter.
|
||||
if delimiter != "" {
|
||||
urlValues.Set("delimiter", delimiter)
|
||||
}
|
||||
|
||||
// maxUploads should be 1000 or less.
|
||||
if maxUploads == 0 || maxUploads > 1000 {
|
||||
maxUploads = 1000
|
||||
}
|
||||
// Set max-uploads.
|
||||
urlValues.Set("max-uploads", fmt.Sprintf("%d", maxUploads))
|
||||
|
||||
|
@ -445,12 +480,15 @@ func (c Client) getTotalMultipartSize(bucketName, objectName, uploadID string) (
|
|||
}
|
||||
|
||||
// listObjectPartsQuery (List Parts query)
|
||||
// - lists some or all (up to 1000) parts that have been uploaded for a specific multipart upload
|
||||
// - lists some or all (up to 1000) parts that have been uploaded
|
||||
// for a specific multipart upload
|
||||
//
|
||||
// You can use the request parameters as selection criteria to return a subset of the uploads in a bucket.
|
||||
// request paramters :-
|
||||
// You can use the request parameters as selection criteria to return
|
||||
// a subset of the uploads in a bucket, request paramters :-
|
||||
// ---------
|
||||
// ?part-number-marker - Specifies the part after which listing should begin.
|
||||
// ?part-number-marker - Specifies the part after which listing should
|
||||
// begin.
|
||||
// ?max-parts - Maximum parts to be listed per request.
|
||||
func (c Client) listObjectPartsQuery(bucketName, objectName, uploadID string, partNumberMarker, maxParts int) (listObjectPartsResult, error) {
|
||||
// Get resources properly escaped and lined up before using them in http request.
|
||||
urlValues := make(url.Values)
|
||||
|
@ -458,6 +496,11 @@ func (c Client) listObjectPartsQuery(bucketName, objectName, uploadID string, pa
|
|||
urlValues.Set("part-number-marker", fmt.Sprintf("%d", partNumberMarker))
|
||||
// Set upload id.
|
||||
urlValues.Set("uploadId", uploadID)
|
||||
|
||||
// maxParts should be 1000 or less.
|
||||
if maxParts == 0 || maxParts > 1000 {
|
||||
maxParts = 1000
|
||||
}
|
||||
// Set max parts.
|
||||
urlValues.Set("max-parts", fmt.Sprintf("%d", maxParts))
|
||||
|
||||
|
|
11
Godeps/_workspace/src/github.com/minio/minio-go/api-presigned.go
generated
vendored
11
Godeps/_workspace/src/github.com/minio/minio-go/api-presigned.go
generated
vendored
|
@ -21,7 +21,7 @@ import (
|
|||
"time"
|
||||
)
|
||||
|
||||
// PresignedGetObject returns a presigned URL to access an object without credentials.
|
||||
// PresignedGetObject - Returns a presigned URL to access an object without credentials.
|
||||
// Expires maximum is 7days - ie. 604800 and minimum is 1.
|
||||
func (c Client) PresignedGetObject(bucketName, objectName string, expires time.Duration) (string, error) {
|
||||
// Input validation.
|
||||
|
@ -50,7 +50,7 @@ func (c Client) PresignedGetObject(bucketName, objectName string, expires time.D
|
|||
return req.URL.String(), nil
|
||||
}
|
||||
|
||||
// PresignedPutObject returns a presigned URL to upload an object without credentials.
|
||||
// PresignedPutObject - Returns a presigned URL to upload an object without credentials.
|
||||
// Expires maximum is 7days - ie. 604800 and minimum is 1.
|
||||
func (c Client) PresignedPutObject(bucketName, objectName string, expires time.Duration) (string, error) {
|
||||
// Input validation.
|
||||
|
@ -79,7 +79,7 @@ func (c Client) PresignedPutObject(bucketName, objectName string, expires time.D
|
|||
return req.URL.String(), nil
|
||||
}
|
||||
|
||||
// PresignedPostPolicy returns POST form data to upload an object at a location.
|
||||
// PresignedPostPolicy - Returns POST form data to upload an object at a location.
|
||||
func (c Client) PresignedPostPolicy(p *PostPolicy) (map[string]string, error) {
|
||||
// Validate input arguments.
|
||||
if p.expiration.IsZero() {
|
||||
|
@ -93,7 +93,7 @@ func (c Client) PresignedPostPolicy(p *PostPolicy) (map[string]string, error) {
|
|||
}
|
||||
|
||||
bucketName := p.formData["bucket"]
|
||||
// Fetch the location.
|
||||
// Fetch the bucket location.
|
||||
location, err := c.getBucketLocation(bucketName)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
|
@ -101,6 +101,7 @@ func (c Client) PresignedPostPolicy(p *PostPolicy) (map[string]string, error) {
|
|||
|
||||
// Keep time.
|
||||
t := time.Now().UTC()
|
||||
// For signature version '2' handle here.
|
||||
if c.signature.isV2() {
|
||||
policyBase64 := p.base64()
|
||||
p.formData["policy"] = policyBase64
|
||||
|
@ -135,7 +136,7 @@ func (c Client) PresignedPostPolicy(p *PostPolicy) (map[string]string, error) {
|
|||
condition: "$x-amz-credential",
|
||||
value: credential,
|
||||
})
|
||||
// get base64 encoded policy.
|
||||
// Get base64 encoded policy.
|
||||
policyBase64 := p.base64()
|
||||
// Fill in the form data.
|
||||
p.formData["policy"] = policyBase64
|
||||
|
|
4
Godeps/_workspace/src/github.com/minio/minio-go/api-put-bucket.go
generated
vendored
4
Godeps/_workspace/src/github.com/minio/minio-go/api-put-bucket.go
generated
vendored
|
@ -97,7 +97,7 @@ func (c Client) makeBucketRequest(bucketName string, acl BucketACL, location str
|
|||
}
|
||||
|
||||
// Set get bucket location always as path style.
|
||||
targetURL := c.endpointURL
|
||||
targetURL := *c.endpointURL
|
||||
if bucketName != "" {
|
||||
// If endpoint supports virtual host style use that always.
|
||||
// Currently only S3 and Google Cloud Storage would support this.
|
||||
|
@ -132,7 +132,7 @@ func (c Client) makeBucketRequest(bucketName string, acl BucketACL, location str
|
|||
|
||||
// If location is not 'us-east-1' create bucket location config.
|
||||
if location != "us-east-1" && location != "" {
|
||||
createBucketConfig := new(createBucketConfiguration)
|
||||
createBucketConfig := createBucketConfiguration{}
|
||||
createBucketConfig.Location = location
|
||||
var createBucketConfigBytes []byte
|
||||
createBucketConfigBytes, err = xml.Marshal(createBucketConfig)
|
||||
|
|
|
@ -28,7 +28,8 @@ import (
|
|||
"sort"
|
||||
)
|
||||
|
||||
// getUploadID if already present for object name or initiate a request to fetch a new upload id.
|
||||
// getUploadID - fetch upload id if already present for an object name
|
||||
// or initiate a new request to fetch a new upload id.
|
||||
func (c Client) getUploadID(bucketName, objectName, contentType string) (string, error) {
|
||||
// Input validation.
|
||||
if err := isValidBucketName(bucketName); err != nil {
|
||||
|
@ -60,84 +61,16 @@ func (c Client) getUploadID(bucketName, objectName, contentType string) (string,
|
|||
return uploadID, nil
|
||||
}
|
||||
|
||||
// FPutObject - put object a file.
|
||||
func (c Client) FPutObject(bucketName, objectName, filePath, contentType string) (int64, error) {
|
||||
// Input validation.
|
||||
if err := isValidBucketName(bucketName); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if err := isValidObjectName(objectName); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
// Open the referenced file.
|
||||
fileData, err := os.Open(filePath)
|
||||
// If any error fail quickly here.
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
defer fileData.Close()
|
||||
|
||||
// Save the file stat.
|
||||
fileStat, err := fileData.Stat()
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
// Save the file size.
|
||||
fileSize := fileStat.Size()
|
||||
if fileSize > int64(maxMultipartPutObjectSize) {
|
||||
return 0, ErrInvalidArgument("Input file size is bigger than the supported maximum of 5TiB.")
|
||||
}
|
||||
|
||||
// NOTE: Google Cloud Storage multipart Put is not compatible with Amazon S3 APIs.
|
||||
// Current implementation will only upload a maximum of 5GiB to Google Cloud Storage servers.
|
||||
if isGoogleEndpoint(c.endpointURL) {
|
||||
if fileSize > int64(maxSinglePutObjectSize) {
|
||||
return 0, ErrorResponse{
|
||||
Code: "NotImplemented",
|
||||
Message: fmt.Sprintf("Invalid Content-Length %d for file uploads to Google Cloud Storage.", fileSize),
|
||||
Key: objectName,
|
||||
BucketName: bucketName,
|
||||
}
|
||||
}
|
||||
// Do not compute MD5 for Google Cloud Storage. Uploads upto 5GiB in size.
|
||||
n, err := c.putNoChecksum(bucketName, objectName, fileData, fileSize, contentType)
|
||||
return n, err
|
||||
}
|
||||
|
||||
// NOTE: S3 doesn't allow anonymous multipart requests.
|
||||
if isAmazonEndpoint(c.endpointURL) && c.anonymous {
|
||||
if fileSize > int64(maxSinglePutObjectSize) {
|
||||
return 0, ErrorResponse{
|
||||
Code: "NotImplemented",
|
||||
Message: fmt.Sprintf("For anonymous requests Content-Length cannot be %d.", fileSize),
|
||||
Key: objectName,
|
||||
BucketName: bucketName,
|
||||
}
|
||||
}
|
||||
// Do not compute MD5 for anonymous requests to Amazon S3. Uploads upto 5GiB in size.
|
||||
n, err := c.putAnonymous(bucketName, objectName, fileData, fileSize, contentType)
|
||||
return n, err
|
||||
}
|
||||
|
||||
// Small object upload is initiated for uploads for input data size smaller than 5MiB.
|
||||
if fileSize < minimumPartSize {
|
||||
return c.putSmallObject(bucketName, objectName, fileData, fileSize, contentType)
|
||||
}
|
||||
return c.fputLargeObject(bucketName, objectName, fileData, fileSize, contentType)
|
||||
}
|
||||
|
||||
// computeHash - calculates MD5 and Sha256 for an input read Seeker.
|
||||
// computeHash - Calculates MD5 and SHA256 for an input read Seeker.
|
||||
func (c Client) computeHash(reader io.ReadSeeker) (md5Sum, sha256Sum []byte, size int64, err error) {
|
||||
// MD5 and Sha256 hasher.
|
||||
var hashMD5, hashSha256 hash.Hash
|
||||
// MD5 and Sha256 hasher.
|
||||
// MD5 and SHA256 hasher.
|
||||
var hashMD5, hashSHA256 hash.Hash
|
||||
// MD5 and SHA256 hasher.
|
||||
hashMD5 = md5.New()
|
||||
hashWriter := io.MultiWriter(hashMD5)
|
||||
if c.signature.isV4() {
|
||||
hashSha256 = sha256.New()
|
||||
hashWriter = io.MultiWriter(hashMD5, hashSha256)
|
||||
hashSHA256 = sha256.New()
|
||||
hashWriter = io.MultiWriter(hashMD5, hashSHA256)
|
||||
}
|
||||
|
||||
size, err = io.Copy(hashWriter, reader)
|
||||
|
@ -153,12 +86,13 @@ func (c Client) computeHash(reader io.ReadSeeker) (md5Sum, sha256Sum []byte, siz
|
|||
// Finalize md5shum and sha256 sum.
|
||||
md5Sum = hashMD5.Sum(nil)
|
||||
if c.signature.isV4() {
|
||||
sha256Sum = hashSha256.Sum(nil)
|
||||
sha256Sum = hashSHA256.Sum(nil)
|
||||
}
|
||||
return md5Sum, sha256Sum, size, nil
|
||||
}
|
||||
|
||||
func (c Client) fputLargeObject(bucketName, objectName string, fileData *os.File, fileSize int64, contentType string) (int64, error) {
|
||||
// FPutObject - Create an object in a bucket, with contents from file at filePath.
|
||||
func (c Client) FPutObject(bucketName, objectName, filePath, contentType string) (n int64, err error) {
|
||||
// Input validation.
|
||||
if err := isValidBucketName(bucketName); err != nil {
|
||||
return 0, err
|
||||
|
@ -167,27 +101,119 @@ func (c Client) fputLargeObject(bucketName, objectName string, fileData *os.File
|
|||
return 0, err
|
||||
}
|
||||
|
||||
// getUploadID for an object, initiates a new multipart request
|
||||
// Open the referenced file.
|
||||
fileReader, err := os.Open(filePath)
|
||||
// If any error fail quickly here.
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
defer fileReader.Close()
|
||||
|
||||
// Save the file stat.
|
||||
fileStat, err := fileReader.Stat()
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
// Save the file size.
|
||||
fileSize := fileStat.Size()
|
||||
|
||||
// Check for largest object size allowed.
|
||||
if fileSize > int64(maxMultipartPutObjectSize) {
|
||||
return 0, ErrEntityTooLarge(fileSize, bucketName, objectName)
|
||||
}
|
||||
|
||||
// NOTE: Google Cloud Storage multipart Put is not compatible with Amazon S3 APIs.
|
||||
// Current implementation will only upload a maximum of 5GiB to Google Cloud Storage servers.
|
||||
if isGoogleEndpoint(c.endpointURL) {
|
||||
if fileSize > int64(maxSinglePutObjectSize) {
|
||||
return 0, ErrorResponse{
|
||||
Code: "NotImplemented",
|
||||
Message: fmt.Sprintf("Invalid Content-Length %d for file uploads to Google Cloud Storage.", fileSize),
|
||||
Key: objectName,
|
||||
BucketName: bucketName,
|
||||
}
|
||||
}
|
||||
// Do not compute MD5 for Google Cloud Storage. Uploads upto 5GiB in size.
|
||||
return c.putObjectNoChecksum(bucketName, objectName, fileReader, fileSize, contentType)
|
||||
}
|
||||
|
||||
// NOTE: S3 doesn't allow anonymous multipart requests.
|
||||
if isAmazonEndpoint(c.endpointURL) && c.anonymous {
|
||||
if fileSize > int64(maxSinglePutObjectSize) {
|
||||
return 0, ErrorResponse{
|
||||
Code: "NotImplemented",
|
||||
Message: fmt.Sprintf("For anonymous requests Content-Length cannot be %d.", fileSize),
|
||||
Key: objectName,
|
||||
BucketName: bucketName,
|
||||
}
|
||||
}
|
||||
// Do not compute MD5 for anonymous requests to Amazon S3. Uploads upto 5GiB in size.
|
||||
return c.putObjectNoChecksum(bucketName, objectName, fileReader, fileSize, contentType)
|
||||
}
|
||||
|
||||
// Small object upload is initiated for uploads for input data size smaller than 5MiB.
|
||||
if fileSize < minimumPartSize {
|
||||
return c.putObjectSingle(bucketName, objectName, fileReader, fileSize, contentType)
|
||||
}
|
||||
// Upload all large objects as multipart.
|
||||
n, err = c.putObjectMultipartFromFile(bucketName, objectName, fileReader, fileSize, contentType)
|
||||
if err != nil {
|
||||
errResp := ToErrorResponse(err)
|
||||
// Verify if multipart functionality is not available, if not
|
||||
// fall back to single PutObject operation.
|
||||
if errResp.Code == "NotImplemented" {
|
||||
// If size of file is greater than '5GiB' fail.
|
||||
if fileSize > maxSinglePutObjectSize {
|
||||
return 0, ErrEntityTooLarge(fileSize, bucketName, objectName)
|
||||
}
|
||||
// Fall back to uploading as single PutObject operation.
|
||||
return c.putObjectSingle(bucketName, objectName, fileReader, fileSize, contentType)
|
||||
}
|
||||
return n, err
|
||||
}
|
||||
return n, nil
|
||||
}
|
||||
|
||||
// putObjectMultipartFromFile - Creates object from contents of *os.File
|
||||
//
|
||||
// NOTE: This function is meant to be used for readers with local
|
||||
// file as in *os.File. This function resumes by skipping all the
|
||||
// necessary parts which were already uploaded by verifying them
|
||||
// against MD5SUM of each individual parts. This function also
|
||||
// effectively utilizes file system capabilities of reading from
|
||||
// specific sections and not having to create temporary files.
|
||||
func (c Client) putObjectMultipartFromFile(bucketName, objectName string, fileReader *os.File, fileSize int64, contentType string) (int64, error) {
|
||||
// Input validation.
|
||||
if err := isValidBucketName(bucketName); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if err := isValidObjectName(objectName); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
// Get upload id for an object, initiates a new multipart request
|
||||
// if it cannot find any previously partially uploaded object.
|
||||
uploadID, err := c.getUploadID(bucketName, objectName, contentType)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
// total data read and written to server. should be equal to 'size' at the end of the call.
|
||||
// Total data read and written to server. should be equal to 'size' at the end of the call.
|
||||
var totalUploadedSize int64
|
||||
|
||||
// Complete multipart upload.
|
||||
var completeMultipartUpload completeMultipartUpload
|
||||
|
||||
// Fetch previously upload parts and save the total size.
|
||||
// Fetch previously upload parts.
|
||||
partsInfo, err := c.listObjectParts(bucketName, objectName, uploadID)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
// Previous maximum part size
|
||||
var prevMaxPartSize int64
|
||||
// Loop through all parts and calculate totalUploadedSize.
|
||||
// Loop through all parts and fetch prevMaxPartSize.
|
||||
for _, partInfo := range partsInfo {
|
||||
// Choose the maximum part size.
|
||||
if partInfo.Size >= prevMaxPartSize {
|
||||
|
@ -197,7 +223,7 @@ func (c Client) fputLargeObject(bucketName, objectName string, fileData *os.File
|
|||
|
||||
// Calculate the optimal part size for a given file size.
|
||||
partSize := optimalPartSize(fileSize)
|
||||
// If prevMaxPartSize is set use that.
|
||||
// Use prevMaxPartSize if available.
|
||||
if prevMaxPartSize != 0 {
|
||||
partSize = prevMaxPartSize
|
||||
}
|
||||
|
@ -205,52 +231,39 @@ func (c Client) fputLargeObject(bucketName, objectName string, fileData *os.File
|
|||
// Part number always starts with '0'.
|
||||
partNumber := 0
|
||||
|
||||
// Loop through until EOF.
|
||||
// Upload each part until fileSize.
|
||||
for totalUploadedSize < fileSize {
|
||||
// Increment part number.
|
||||
partNumber++
|
||||
|
||||
// Get a section reader on a particular offset.
|
||||
sectionReader := io.NewSectionReader(fileData, totalUploadedSize, partSize)
|
||||
sectionReader := io.NewSectionReader(fileReader, totalUploadedSize, partSize)
|
||||
|
||||
// Calculates MD5 and Sha256 sum for a section reader.
|
||||
// Calculates MD5 and SHA256 sum for a section reader.
|
||||
md5Sum, sha256Sum, size, err := c.computeHash(sectionReader)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
// Save all the part metadata.
|
||||
prtData := partData{
|
||||
ReadCloser: ioutil.NopCloser(sectionReader),
|
||||
Size: size,
|
||||
MD5Sum: md5Sum,
|
||||
Sha256Sum: sha256Sum,
|
||||
Number: partNumber, // Part number to be uploaded.
|
||||
}
|
||||
|
||||
// If part not uploaded proceed to upload.
|
||||
// Verify if part was not uploaded.
|
||||
if !isPartUploaded(objectPart{
|
||||
ETag: hex.EncodeToString(prtData.MD5Sum),
|
||||
PartNumber: prtData.Number,
|
||||
ETag: hex.EncodeToString(md5Sum),
|
||||
PartNumber: partNumber,
|
||||
}, partsInfo) {
|
||||
// Upload the part.
|
||||
objPart, err := c.uploadPart(bucketName, objectName, uploadID, prtData)
|
||||
// Proceed to upload the part.
|
||||
objPart, err := c.uploadPart(bucketName, objectName, uploadID, ioutil.NopCloser(sectionReader), partNumber, md5Sum, sha256Sum, size)
|
||||
if err != nil {
|
||||
prtData.ReadCloser.Close()
|
||||
return totalUploadedSize, err
|
||||
}
|
||||
// Save successfully uploaded part metadata.
|
||||
partsInfo[prtData.Number] = objPart
|
||||
partsInfo[partNumber] = objPart
|
||||
}
|
||||
|
||||
// Close the read closer for temporary file.
|
||||
prtData.ReadCloser.Close()
|
||||
|
||||
// Save successfully uploaded size.
|
||||
totalUploadedSize += prtData.Size
|
||||
totalUploadedSize += size
|
||||
}
|
||||
|
||||
// if totalUploadedSize is different than the file 'size'. Do not complete the request throw an error.
|
||||
// Verify if we uploaded all data.
|
||||
if totalUploadedSize != fileSize {
|
||||
return totalUploadedSize, ErrUnexpectedEOF(totalUploadedSize, fileSize, bucketName, objectName)
|
||||
}
|
||||
|
@ -263,7 +276,7 @@ func (c Client) fputLargeObject(bucketName, objectName string, fileData *os.File
|
|||
completeMultipartUpload.Parts = append(completeMultipartUpload.Parts, complPart)
|
||||
}
|
||||
|
||||
// If partNumber is different than total list of parts, error out.
|
||||
// Verify if partNumber is different than total list of parts.
|
||||
if partNumber != len(completeMultipartUpload.Parts) {
|
||||
return totalUploadedSize, ErrInvalidParts(partNumber, len(completeMultipartUpload.Parts))
|
||||
}
|
421
Godeps/_workspace/src/github.com/minio/minio-go/api-put-object-multipart.go
generated
vendored
Normal file
421
Godeps/_workspace/src/github.com/minio/minio-go/api-put-object-multipart.go
generated
vendored
Normal file
|
@ -0,0 +1,421 @@
|
|||
/*
|
||||
* Minio Go Library for Amazon S3 Compatible Cloud Storage (C) 2015 Minio, Inc.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package minio
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"crypto/md5"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/xml"
|
||||
"hash"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// Verify if reader is *os.File
|
||||
func isFile(reader io.Reader) (ok bool) {
|
||||
_, ok = reader.(*os.File)
|
||||
return
|
||||
}
|
||||
|
||||
// Verify if reader is *minio.Object
|
||||
func isObject(reader io.Reader) (ok bool) {
|
||||
_, ok = reader.(*Object)
|
||||
return
|
||||
}
|
||||
|
||||
// Verify if reader is a generic ReaderAt
|
||||
func isReadAt(reader io.Reader) (ok bool) {
|
||||
_, ok = reader.(io.ReaderAt)
|
||||
return
|
||||
}
|
||||
|
||||
// hashCopyN - Calculates Md5sum and SHA256sum for upto partSize amount of bytes.
|
||||
func (c Client) hashCopyN(writer io.ReadWriteSeeker, reader io.Reader, partSize int64) (md5Sum, sha256Sum []byte, size int64, err error) {
|
||||
// MD5 and SHA256 hasher.
|
||||
var hashMD5, hashSHA256 hash.Hash
|
||||
// MD5 and SHA256 hasher.
|
||||
hashMD5 = md5.New()
|
||||
hashWriter := io.MultiWriter(writer, hashMD5)
|
||||
if c.signature.isV4() {
|
||||
hashSHA256 = sha256.New()
|
||||
hashWriter = io.MultiWriter(writer, hashMD5, hashSHA256)
|
||||
}
|
||||
|
||||
// Copies to input at writer.
|
||||
size, err = io.CopyN(hashWriter, reader, partSize)
|
||||
if err != nil {
|
||||
// If not EOF return error right here.
|
||||
if err != io.EOF {
|
||||
return nil, nil, 0, err
|
||||
}
|
||||
}
|
||||
|
||||
// Seek back to beginning of input, any error fail right here.
|
||||
if _, err := writer.Seek(0, 0); err != nil {
|
||||
return nil, nil, 0, err
|
||||
}
|
||||
|
||||
// Finalize md5shum and sha256 sum.
|
||||
md5Sum = hashMD5.Sum(nil)
|
||||
if c.signature.isV4() {
|
||||
sha256Sum = hashSHA256.Sum(nil)
|
||||
}
|
||||
return md5Sum, sha256Sum, size, err
|
||||
}
|
||||
|
||||
// Comprehensive put object operation involving multipart resumable uploads.
|
||||
//
|
||||
// Following code handles these types of readers.
|
||||
//
|
||||
// - *os.File
|
||||
// - *minio.Object
|
||||
// - Any reader which has a method 'ReadAt()'
|
||||
//
|
||||
// If we exhaust all the known types, code proceeds to use stream as
|
||||
// is where each part is re-downloaded, checksummed and verified
|
||||
// before upload.
|
||||
func (c Client) putObjectMultipart(bucketName, objectName string, reader io.Reader, size int64, contentType string) (n int64, err error) {
|
||||
if size > 0 && size >= minimumPartSize {
|
||||
// Verify if reader is *os.File, then use file system functionalities.
|
||||
if isFile(reader) {
|
||||
return c.putObjectMultipartFromFile(bucketName, objectName, reader.(*os.File), size, contentType)
|
||||
}
|
||||
// Verify if reader is *minio.Object or io.ReaderAt.
|
||||
// NOTE: Verification of object is kept for a specific purpose
|
||||
// while it is going to be duck typed similar to io.ReaderAt.
|
||||
// It is to indicate that *minio.Object implements io.ReaderAt.
|
||||
// and such a functionality is used in the subsequent code
|
||||
// path.
|
||||
if isObject(reader) || isReadAt(reader) {
|
||||
return c.putObjectMultipartFromReadAt(bucketName, objectName, reader.(io.ReaderAt), size, contentType)
|
||||
}
|
||||
}
|
||||
// For any other data size and reader type we do generic multipart
|
||||
// approach by staging data in temporary files and uploading them.
|
||||
return c.putObjectMultipartStream(bucketName, objectName, reader, size, contentType)
|
||||
}
|
||||
|
||||
// putObjectStream uploads files bigger than 5MiB, and also supports
|
||||
// special case where size is unknown i.e '-1'.
|
||||
func (c Client) putObjectMultipartStream(bucketName, objectName string, reader io.Reader, size int64, contentType string) (n int64, err error) {
|
||||
// Input validation.
|
||||
if err := isValidBucketName(bucketName); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if err := isValidObjectName(objectName); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
// getUploadID for an object, initiates a new multipart request
|
||||
// if it cannot find any previously partially uploaded object.
|
||||
uploadID, err := c.getUploadID(bucketName, objectName, contentType)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
// Total data read and written to server. should be equal to 'size' at the end of the call.
|
||||
var totalUploadedSize int64
|
||||
|
||||
// Complete multipart upload.
|
||||
var completeMultipartUpload completeMultipartUpload
|
||||
|
||||
// Fetch previously upload parts.
|
||||
partsInfo, err := c.listObjectParts(bucketName, objectName, uploadID)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
// Previous maximum part size
|
||||
var prevMaxPartSize int64
|
||||
// Loop through all parts and calculate totalUploadedSize.
|
||||
for _, partInfo := range partsInfo {
|
||||
// Choose the maximum part size.
|
||||
if partInfo.Size >= prevMaxPartSize {
|
||||
prevMaxPartSize = partInfo.Size
|
||||
}
|
||||
}
|
||||
|
||||
// Calculate the optimal part size for a given size.
|
||||
partSize := optimalPartSize(size)
|
||||
// Use prevMaxPartSize if available.
|
||||
if prevMaxPartSize != 0 {
|
||||
partSize = prevMaxPartSize
|
||||
}
|
||||
|
||||
// Part number always starts with '0'.
|
||||
partNumber := 0
|
||||
|
||||
// Upload each part until EOF.
|
||||
for {
|
||||
// Increment part number.
|
||||
partNumber++
|
||||
|
||||
// Initialize a new temporary file.
|
||||
tmpFile, err := newTempFile("multiparts$-putobject-stream")
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
// Calculates MD5 and SHA256 sum while copying partSize bytes into tmpFile.
|
||||
md5Sum, sha256Sum, size, rErr := c.hashCopyN(tmpFile, reader, partSize)
|
||||
if rErr != nil {
|
||||
if rErr != io.EOF {
|
||||
return 0, rErr
|
||||
}
|
||||
}
|
||||
|
||||
// Verify if part was not uploaded.
|
||||
if !isPartUploaded(objectPart{
|
||||
ETag: hex.EncodeToString(md5Sum),
|
||||
PartNumber: partNumber,
|
||||
}, partsInfo) {
|
||||
// Proceed to upload the part.
|
||||
objPart, err := c.uploadPart(bucketName, objectName, uploadID, tmpFile, partNumber, md5Sum, sha256Sum, size)
|
||||
if err != nil {
|
||||
// Close the temporary file upon any error.
|
||||
tmpFile.Close()
|
||||
return 0, err
|
||||
}
|
||||
// Save successfully uploaded part metadata.
|
||||
partsInfo[partNumber] = objPart
|
||||
}
|
||||
|
||||
// Close the temporary file.
|
||||
tmpFile.Close()
|
||||
|
||||
// If read error was an EOF, break out of the loop.
|
||||
if rErr == io.EOF {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
// Verify if we uploaded all the data.
|
||||
if size > 0 {
|
||||
if totalUploadedSize != size {
|
||||
return totalUploadedSize, ErrUnexpectedEOF(totalUploadedSize, size, bucketName, objectName)
|
||||
}
|
||||
}
|
||||
|
||||
// Loop over uploaded parts to save them in a Parts array before completing the multipart request.
|
||||
for _, part := range partsInfo {
|
||||
var complPart completePart
|
||||
complPart.ETag = part.ETag
|
||||
complPart.PartNumber = part.PartNumber
|
||||
completeMultipartUpload.Parts = append(completeMultipartUpload.Parts, complPart)
|
||||
// Save successfully uploaded size.
|
||||
totalUploadedSize += part.Size
|
||||
}
|
||||
|
||||
// Verify if partNumber is different than total list of parts.
|
||||
if partNumber != len(completeMultipartUpload.Parts) {
|
||||
return totalUploadedSize, ErrInvalidParts(partNumber, len(completeMultipartUpload.Parts))
|
||||
}
|
||||
|
||||
// Sort all completed parts.
|
||||
sort.Sort(completedParts(completeMultipartUpload.Parts))
|
||||
_, err = c.completeMultipartUpload(bucketName, objectName, uploadID, completeMultipartUpload)
|
||||
if err != nil {
|
||||
return totalUploadedSize, err
|
||||
}
|
||||
|
||||
// Return final size.
|
||||
return totalUploadedSize, nil
|
||||
}
|
||||
|
||||
// initiateMultipartUpload - Initiates a multipart upload and returns an upload ID.
|
||||
func (c Client) initiateMultipartUpload(bucketName, objectName, contentType string) (initiateMultipartUploadResult, error) {
|
||||
// Input validation.
|
||||
if err := isValidBucketName(bucketName); err != nil {
|
||||
return initiateMultipartUploadResult{}, err
|
||||
}
|
||||
if err := isValidObjectName(objectName); err != nil {
|
||||
return initiateMultipartUploadResult{}, err
|
||||
}
|
||||
|
||||
// Initialize url queries.
|
||||
urlValues := make(url.Values)
|
||||
urlValues.Set("uploads", "")
|
||||
|
||||
if contentType == "" {
|
||||
contentType = "application/octet-stream"
|
||||
}
|
||||
|
||||
// Set ContentType header.
|
||||
customHeader := make(http.Header)
|
||||
customHeader.Set("Content-Type", contentType)
|
||||
|
||||
reqMetadata := requestMetadata{
|
||||
bucketName: bucketName,
|
||||
objectName: objectName,
|
||||
queryValues: urlValues,
|
||||
customHeader: customHeader,
|
||||
}
|
||||
|
||||
// Instantiate the request.
|
||||
req, err := c.newRequest("POST", reqMetadata)
|
||||
if err != nil {
|
||||
return initiateMultipartUploadResult{}, err
|
||||
}
|
||||
|
||||
// Execute the request.
|
||||
resp, err := c.do(req)
|
||||
defer closeResponse(resp)
|
||||
if err != nil {
|
||||
return initiateMultipartUploadResult{}, err
|
||||
}
|
||||
if resp != nil {
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return initiateMultipartUploadResult{}, HTTPRespToErrorResponse(resp, bucketName, objectName)
|
||||
}
|
||||
}
|
||||
// Decode xml for new multipart upload.
|
||||
initiateMultipartUploadResult := initiateMultipartUploadResult{}
|
||||
err = xmlDecoder(resp.Body, &initiateMultipartUploadResult)
|
||||
if err != nil {
|
||||
return initiateMultipartUploadResult, err
|
||||
}
|
||||
return initiateMultipartUploadResult, nil
|
||||
}
|
||||
|
||||
// uploadPart - Uploads a part in a multipart upload.
|
||||
func (c Client) uploadPart(bucketName, objectName, uploadID string, reader io.ReadCloser, partNumber int, md5Sum, sha256Sum []byte, size int64) (objectPart, error) {
|
||||
// Input validation.
|
||||
if err := isValidBucketName(bucketName); err != nil {
|
||||
return objectPart{}, err
|
||||
}
|
||||
if err := isValidObjectName(objectName); err != nil {
|
||||
return objectPart{}, err
|
||||
}
|
||||
if size > maxPartSize {
|
||||
return objectPart{}, ErrEntityTooLarge(size, bucketName, objectName)
|
||||
}
|
||||
if size <= -1 {
|
||||
return objectPart{}, ErrEntityTooSmall(size, bucketName, objectName)
|
||||
}
|
||||
if partNumber <= 0 {
|
||||
return objectPart{}, ErrInvalidArgument("Part number cannot be negative or equal to zero.")
|
||||
}
|
||||
if uploadID == "" {
|
||||
return objectPart{}, ErrInvalidArgument("UploadID cannot be empty.")
|
||||
}
|
||||
|
||||
// Get resources properly escaped and lined up before using them in http request.
|
||||
urlValues := make(url.Values)
|
||||
// Set part number.
|
||||
urlValues.Set("partNumber", strconv.Itoa(partNumber))
|
||||
// Set upload id.
|
||||
urlValues.Set("uploadId", uploadID)
|
||||
|
||||
reqMetadata := requestMetadata{
|
||||
bucketName: bucketName,
|
||||
objectName: objectName,
|
||||
queryValues: urlValues,
|
||||
contentBody: reader,
|
||||
contentLength: size,
|
||||
contentMD5Bytes: md5Sum,
|
||||
contentSHA256Bytes: sha256Sum,
|
||||
}
|
||||
|
||||
// Instantiate a request.
|
||||
req, err := c.newRequest("PUT", reqMetadata)
|
||||
if err != nil {
|
||||
return objectPart{}, err
|
||||
}
|
||||
// Execute the request.
|
||||
resp, err := c.do(req)
|
||||
defer closeResponse(resp)
|
||||
if err != nil {
|
||||
return objectPart{}, err
|
||||
}
|
||||
if resp != nil {
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return objectPart{}, HTTPRespToErrorResponse(resp, bucketName, objectName)
|
||||
}
|
||||
}
|
||||
// Once successfully uploaded, return completed part.
|
||||
objPart := objectPart{}
|
||||
objPart.Size = size
|
||||
objPart.PartNumber = partNumber
|
||||
// Trim off the odd double quotes from ETag in the beginning and end.
|
||||
objPart.ETag = strings.TrimPrefix(resp.Header.Get("ETag"), "\"")
|
||||
objPart.ETag = strings.TrimSuffix(objPart.ETag, "\"")
|
||||
return objPart, nil
|
||||
}
|
||||
|
||||
// completeMultipartUpload - Completes a multipart upload by assembling previously uploaded parts.
|
||||
func (c Client) completeMultipartUpload(bucketName, objectName, uploadID string, complete completeMultipartUpload) (completeMultipartUploadResult, error) {
|
||||
// Input validation.
|
||||
if err := isValidBucketName(bucketName); err != nil {
|
||||
return completeMultipartUploadResult{}, err
|
||||
}
|
||||
if err := isValidObjectName(objectName); err != nil {
|
||||
return completeMultipartUploadResult{}, err
|
||||
}
|
||||
|
||||
// Initialize url queries.
|
||||
urlValues := make(url.Values)
|
||||
urlValues.Set("uploadId", uploadID)
|
||||
|
||||
// Marshal complete multipart body.
|
||||
completeMultipartUploadBytes, err := xml.Marshal(complete)
|
||||
if err != nil {
|
||||
return completeMultipartUploadResult{}, err
|
||||
}
|
||||
|
||||
// Instantiate all the complete multipart buffer.
|
||||
completeMultipartUploadBuffer := bytes.NewBuffer(completeMultipartUploadBytes)
|
||||
reqMetadata := requestMetadata{
|
||||
bucketName: bucketName,
|
||||
objectName: objectName,
|
||||
queryValues: urlValues,
|
||||
contentBody: ioutil.NopCloser(completeMultipartUploadBuffer),
|
||||
contentLength: int64(completeMultipartUploadBuffer.Len()),
|
||||
contentSHA256Bytes: sum256(completeMultipartUploadBuffer.Bytes()),
|
||||
}
|
||||
|
||||
// Instantiate the request.
|
||||
req, err := c.newRequest("POST", reqMetadata)
|
||||
if err != nil {
|
||||
return completeMultipartUploadResult{}, err
|
||||
}
|
||||
|
||||
// Execute the request.
|
||||
resp, err := c.do(req)
|
||||
defer closeResponse(resp)
|
||||
if err != nil {
|
||||
return completeMultipartUploadResult{}, err
|
||||
}
|
||||
if resp != nil {
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return completeMultipartUploadResult{}, HTTPRespToErrorResponse(resp, bucketName, objectName)
|
||||
}
|
||||
}
|
||||
// Decode completed multipart upload response on success.
|
||||
completeMultipartUploadResult := completeMultipartUploadResult{}
|
||||
err = xmlDecoder(resp.Body, &completeMultipartUploadResult)
|
||||
if err != nil {
|
||||
return completeMultipartUploadResult, err
|
||||
}
|
||||
return completeMultipartUploadResult, nil
|
||||
}
|
378
Godeps/_workspace/src/github.com/minio/minio-go/api-put-object-partial.go
generated
vendored
378
Godeps/_workspace/src/github.com/minio/minio-go/api-put-object-partial.go
generated
vendored
|
@ -1,378 +0,0 @@
|
|||
/*
|
||||
* Minio Go Library for Amazon S3 Compatible Cloud Storage (C) 2015 Minio, Inc.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package minio
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"crypto/md5"
|
||||
"crypto/sha256"
|
||||
"errors"
|
||||
"fmt"
|
||||
"hash"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"sort"
|
||||
)
|
||||
|
||||
// PutObjectPartial put object partial.
|
||||
func (c Client) PutObjectPartial(bucketName, objectName string, data ReadAtCloser, size int64, contentType string) (n int64, err error) {
|
||||
// Input validation.
|
||||
if err := isValidBucketName(bucketName); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if err := isValidObjectName(objectName); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
// Input size negative should return error.
|
||||
if size < 0 {
|
||||
return 0, ErrInvalidArgument("Input file size cannot be negative.")
|
||||
}
|
||||
// Input size bigger than 5TiB should fail.
|
||||
if size > int64(maxMultipartPutObjectSize) {
|
||||
return 0, ErrInvalidArgument("Input file size is bigger than the supported maximum of 5TiB.")
|
||||
}
|
||||
|
||||
// NOTE: Google Cloud Storage does not implement Amazon S3 Compatible multipart PUT.
|
||||
// So we fall back to single PUT operation with the maximum limit of 5GiB.
|
||||
if isGoogleEndpoint(c.endpointURL) {
|
||||
if size > int64(maxSinglePutObjectSize) {
|
||||
return 0, ErrorResponse{
|
||||
Code: "NotImplemented",
|
||||
Message: fmt.Sprintf("Invalid Content-Length %d for file uploads to Google Cloud Storage.", size),
|
||||
Key: objectName,
|
||||
BucketName: bucketName,
|
||||
}
|
||||
}
|
||||
// Do not compute MD5 for Google Cloud Storage. Uploads upto 5GiB in size.
|
||||
n, err := c.putPartialNoChksum(bucketName, objectName, data, size, contentType)
|
||||
return n, err
|
||||
}
|
||||
|
||||
// NOTE: S3 doesn't allow anonymous multipart requests.
|
||||
if isAmazonEndpoint(c.endpointURL) && c.anonymous {
|
||||
if size > int64(maxSinglePutObjectSize) {
|
||||
return 0, ErrorResponse{
|
||||
Code: "NotImplemented",
|
||||
Message: fmt.Sprintf("For anonymous requests Content-Length cannot be %d.", size),
|
||||
Key: objectName,
|
||||
BucketName: bucketName,
|
||||
}
|
||||
}
|
||||
// Do not compute MD5 for anonymous requests to Amazon S3. Uploads upto 5GiB in size.
|
||||
n, err := c.putPartialAnonymous(bucketName, objectName, data, size, contentType)
|
||||
return n, err
|
||||
}
|
||||
|
||||
// Small file upload is initiated for uploads for input data size smaller than 5MiB.
|
||||
if size < minimumPartSize {
|
||||
n, err = c.putPartialSmallObject(bucketName, objectName, data, size, contentType)
|
||||
return n, err
|
||||
}
|
||||
n, err = c.putPartialLargeObject(bucketName, objectName, data, size, contentType)
|
||||
return n, err
|
||||
|
||||
}
|
||||
|
||||
// putNoChecksumPartial special function used Google Cloud Storage. This special function
|
||||
// is used for Google Cloud Storage since Google's multipart API is not S3 compatible.
|
||||
func (c Client) putPartialNoChksum(bucketName, objectName string, data ReadAtCloser, size int64, contentType string) (n int64, err error) {
|
||||
// Input validation.
|
||||
if err := isValidBucketName(bucketName); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if err := isValidObjectName(objectName); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if size > maxPartSize {
|
||||
return 0, ErrEntityTooLarge(size, bucketName, objectName)
|
||||
}
|
||||
|
||||
// Create a new pipe to stage the reads.
|
||||
reader, writer := io.Pipe()
|
||||
|
||||
// readAtOffset to carry future offsets.
|
||||
var readAtOffset int64
|
||||
|
||||
// readAt defaults to reading at 5MiB buffer.
|
||||
readAtBuffer := make([]byte, 1024*1024*5)
|
||||
|
||||
// Initiate a routine to start writing.
|
||||
go func() {
|
||||
for {
|
||||
readAtSize, rerr := data.ReadAt(readAtBuffer, readAtOffset)
|
||||
if rerr != nil {
|
||||
if rerr != io.EOF {
|
||||
writer.CloseWithError(rerr)
|
||||
return
|
||||
}
|
||||
}
|
||||
writeSize, werr := writer.Write(readAtBuffer[:readAtSize])
|
||||
if werr != nil {
|
||||
writer.CloseWithError(werr)
|
||||
return
|
||||
}
|
||||
if readAtSize != writeSize {
|
||||
writer.CloseWithError(errors.New("Something really bad happened here. " + reportIssue))
|
||||
return
|
||||
}
|
||||
readAtOffset += int64(writeSize)
|
||||
if rerr == io.EOF {
|
||||
writer.Close()
|
||||
return
|
||||
}
|
||||
}
|
||||
}()
|
||||
// For anonymous requests, we will not calculate sha256 and md5sum.
|
||||
putObjData := putObjectData{
|
||||
MD5Sum: nil,
|
||||
Sha256Sum: nil,
|
||||
ReadCloser: reader,
|
||||
Size: size,
|
||||
ContentType: contentType,
|
||||
}
|
||||
// Execute put object.
|
||||
st, err := c.putObject(bucketName, objectName, putObjData)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if st.Size != size {
|
||||
return 0, ErrUnexpectedEOF(st.Size, size, bucketName, objectName)
|
||||
}
|
||||
return size, nil
|
||||
}
|
||||
|
||||
// putAnonymousPartial is a special function for uploading content as anonymous request.
|
||||
// This special function is necessary since Amazon S3 doesn't allow anonymous multipart uploads.
|
||||
func (c Client) putPartialAnonymous(bucketName, objectName string, data ReadAtCloser, size int64, contentType string) (n int64, err error) {
|
||||
// Input validation.
|
||||
if err := isValidBucketName(bucketName); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if err := isValidObjectName(objectName); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return c.putPartialNoChksum(bucketName, objectName, data, size, contentType)
|
||||
}
|
||||
|
||||
// putSmallObjectPartial uploads files smaller than 5MiB.
|
||||
func (c Client) putPartialSmallObject(bucketName, objectName string, data ReadAtCloser, size int64, contentType string) (n int64, err error) {
|
||||
// Input validation.
|
||||
if err := isValidBucketName(bucketName); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if err := isValidObjectName(objectName); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
// readAt defaults to reading at 5MiB buffer.
|
||||
readAtBuffer := make([]byte, size)
|
||||
readAtSize, err := data.ReadAt(readAtBuffer, 0)
|
||||
if err != nil {
|
||||
if err != io.EOF {
|
||||
return 0, err
|
||||
}
|
||||
}
|
||||
if int64(readAtSize) != size {
|
||||
return 0, ErrUnexpectedEOF(int64(readAtSize), size, bucketName, objectName)
|
||||
}
|
||||
|
||||
// Construct a new PUT object metadata.
|
||||
putObjData := putObjectData{
|
||||
MD5Sum: sumMD5(readAtBuffer),
|
||||
Sha256Sum: sum256(readAtBuffer),
|
||||
ReadCloser: ioutil.NopCloser(bytes.NewReader(readAtBuffer)),
|
||||
Size: size,
|
||||
ContentType: contentType,
|
||||
}
|
||||
// Single part use case, use putObject directly.
|
||||
st, err := c.putObject(bucketName, objectName, putObjData)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if st.Size != size {
|
||||
return 0, ErrUnexpectedEOF(st.Size, size, bucketName, objectName)
|
||||
}
|
||||
return size, nil
|
||||
}
|
||||
|
||||
// putPartialLargeObject uploads files bigger than 5MiB.
|
||||
func (c Client) putPartialLargeObject(bucketName, objectName string, data ReadAtCloser, size int64, contentType string) (n int64, err error) {
|
||||
// Input validation.
|
||||
if err := isValidBucketName(bucketName); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if err := isValidObjectName(objectName); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
// getUploadID for an object, initiates a new multipart request
|
||||
// if it cannot find any previously partially uploaded object.
|
||||
uploadID, err := c.getUploadID(bucketName, objectName, contentType)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
// total data read and written to server. should be equal to 'size' at the end of the call.
|
||||
var totalUploadedSize int64
|
||||
|
||||
// Complete multipart upload.
|
||||
var completeMultipartUpload completeMultipartUpload
|
||||
|
||||
// Fetch previously upload parts and save the total size.
|
||||
partsInfo, err := c.listObjectParts(bucketName, objectName, uploadID)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
// Previous maximum part size
|
||||
var prevMaxPartSize int64
|
||||
// previous part number.
|
||||
var prevPartNumber int
|
||||
// Loop through all parts and calculate totalUploadedSize.
|
||||
for _, partInfo := range partsInfo {
|
||||
totalUploadedSize += partInfo.Size
|
||||
// Choose the maximum part size.
|
||||
if partInfo.Size >= prevMaxPartSize {
|
||||
prevMaxPartSize = partInfo.Size
|
||||
}
|
||||
// Save previous part number.
|
||||
prevPartNumber = partInfo.PartNumber
|
||||
}
|
||||
|
||||
// Calculate the optimal part size for a given file size.
|
||||
partSize := optimalPartSize(size)
|
||||
// If prevMaxPartSize is set use that.
|
||||
if prevMaxPartSize != 0 {
|
||||
partSize = prevMaxPartSize
|
||||
}
|
||||
|
||||
// MD5 and Sha256 hasher.
|
||||
var hashMD5, hashSha256 hash.Hash
|
||||
|
||||
// Part number always starts with prevPartNumber + 1. i.e The next part number.
|
||||
partNumber := prevPartNumber + 1
|
||||
|
||||
// Loop through until EOF.
|
||||
for totalUploadedSize < size {
|
||||
// Initialize a new temporary file.
|
||||
tmpFile, err := newTempFile("multiparts$-putobject-partial")
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
// Create a hash multiwriter.
|
||||
hashMD5 = md5.New()
|
||||
hashWriter := io.MultiWriter(hashMD5)
|
||||
if c.signature.isV4() {
|
||||
hashSha256 = sha256.New()
|
||||
hashWriter = io.MultiWriter(hashMD5, hashSha256)
|
||||
}
|
||||
writer := io.MultiWriter(tmpFile, hashWriter)
|
||||
|
||||
// totalUploadedSize is the current readAtOffset.
|
||||
readAtOffset := totalUploadedSize
|
||||
|
||||
// Read until partSize.
|
||||
var totalReadPartSize int64
|
||||
|
||||
// readAt defaults to reading at 5MiB buffer.
|
||||
readAtBuffer := make([]byte, optimalReadAtBufferSize)
|
||||
|
||||
// Loop through until partSize.
|
||||
for totalReadPartSize < partSize {
|
||||
readAtSize, rerr := data.ReadAt(readAtBuffer, readAtOffset)
|
||||
if rerr != nil {
|
||||
if rerr != io.EOF {
|
||||
return 0, rerr
|
||||
}
|
||||
}
|
||||
writeSize, werr := writer.Write(readAtBuffer[:readAtSize])
|
||||
if werr != nil {
|
||||
return 0, werr
|
||||
}
|
||||
if readAtSize != writeSize {
|
||||
return 0, errors.New("Something really bad happened here. " + reportIssue)
|
||||
}
|
||||
readAtOffset += int64(writeSize)
|
||||
totalReadPartSize += int64(writeSize)
|
||||
if rerr == io.EOF {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
// Seek back to beginning of the temporary file.
|
||||
if _, err := tmpFile.Seek(0, 0); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
// Save all the part metadata.
|
||||
prtData := partData{
|
||||
ReadCloser: tmpFile,
|
||||
MD5Sum: hashMD5.Sum(nil),
|
||||
Size: totalReadPartSize,
|
||||
}
|
||||
|
||||
// Signature version '4'.
|
||||
if c.signature.isV4() {
|
||||
prtData.Sha256Sum = hashSha256.Sum(nil)
|
||||
}
|
||||
|
||||
// Current part number to be uploaded.
|
||||
prtData.Number = partNumber
|
||||
|
||||
// execute upload part.
|
||||
objPart, err := c.uploadPart(bucketName, objectName, uploadID, prtData)
|
||||
if err != nil {
|
||||
// Close the read closer.
|
||||
prtData.ReadCloser.Close()
|
||||
return totalUploadedSize, err
|
||||
}
|
||||
|
||||
// Save successfully uploaded size.
|
||||
totalUploadedSize += prtData.Size
|
||||
|
||||
// Save successfully uploaded part metadata.
|
||||
partsInfo[prtData.Number] = objPart
|
||||
|
||||
// Move to next part.
|
||||
partNumber++
|
||||
}
|
||||
|
||||
// If size is greater than zero verify totalUploaded.
|
||||
// if totalUploaded is different than the input 'size', do not complete the request throw an error.
|
||||
if totalUploadedSize != size {
|
||||
return totalUploadedSize, ErrUnexpectedEOF(totalUploadedSize, size, bucketName, objectName)
|
||||
}
|
||||
|
||||
// Loop over uploaded parts to save them in a Parts array before completing the multipart request.
|
||||
for _, part := range partsInfo {
|
||||
var complPart completePart
|
||||
complPart.ETag = part.ETag
|
||||
complPart.PartNumber = part.PartNumber
|
||||
completeMultipartUpload.Parts = append(completeMultipartUpload.Parts, complPart)
|
||||
}
|
||||
|
||||
// Sort all completed parts.
|
||||
sort.Sort(completedParts(completeMultipartUpload.Parts))
|
||||
_, err = c.completeMultipartUpload(bucketName, objectName, uploadID, completeMultipartUpload)
|
||||
if err != nil {
|
||||
return totalUploadedSize, err
|
||||
}
|
||||
|
||||
// Return final size.
|
||||
return totalUploadedSize, nil
|
||||
}
|
196
Godeps/_workspace/src/github.com/minio/minio-go/api-put-object-readat.go
generated
vendored
Normal file
196
Godeps/_workspace/src/github.com/minio/minio-go/api-put-object-readat.go
generated
vendored
Normal file
|
@ -0,0 +1,196 @@
|
|||
/*
|
||||
* Minio Go Library for Amazon S3 Compatible Cloud Storage (C) 2015 Minio, Inc.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package minio
|
||||
|
||||
import (
|
||||
"crypto/md5"
|
||||
"crypto/sha256"
|
||||
"errors"
|
||||
"hash"
|
||||
"io"
|
||||
"sort"
|
||||
)
|
||||
|
||||
// putObjectMultipartFromReadAt - Uploads files bigger than 5MiB. Supports reader
|
||||
// of type which implements io.ReaderAt interface (ReadAt method).
|
||||
//
|
||||
// NOTE: This function is meant to be used for all readers which
|
||||
// implement io.ReaderAt which allows us for resuming multipart
|
||||
// uploads but reading at an offset, which would avoid re-read the
|
||||
// data which was already uploaded. Internally this function uses
|
||||
// temporary files for staging all the data, these temporary files are
|
||||
// cleaned automatically when the caller i.e http client closes the
|
||||
// stream after uploading all the contents successfully.
|
||||
func (c Client) putObjectMultipartFromReadAt(bucketName, objectName string, reader io.ReaderAt, size int64, contentType string) (n int64, err error) {
|
||||
// Input validation.
|
||||
if err := isValidBucketName(bucketName); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if err := isValidObjectName(objectName); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
// Get upload id for an object, initiates a new multipart request
|
||||
// if it cannot find any previously partially uploaded object.
|
||||
uploadID, err := c.getUploadID(bucketName, objectName, contentType)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
// Total data read and written to server. should be equal to 'size' at the end of the call.
|
||||
var totalUploadedSize int64
|
||||
|
||||
// Complete multipart upload.
|
||||
var completeMultipartUpload completeMultipartUpload
|
||||
|
||||
// Fetch previously upload parts.
|
||||
partsInfo, err := c.listObjectParts(bucketName, objectName, uploadID)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
// Previous maximum part size
|
||||
var prevMaxPartSize int64
|
||||
// Previous part number.
|
||||
var prevPartNumber int
|
||||
// Loop through all parts and calculate totalUploadedSize.
|
||||
for _, partInfo := range partsInfo {
|
||||
totalUploadedSize += partInfo.Size
|
||||
// Choose the maximum part size.
|
||||
if partInfo.Size >= prevMaxPartSize {
|
||||
prevMaxPartSize = partInfo.Size
|
||||
}
|
||||
// Save previous part number.
|
||||
prevPartNumber = partInfo.PartNumber
|
||||
}
|
||||
|
||||
// Calculate the optimal part size for a given file size.
|
||||
partSize := optimalPartSize(size)
|
||||
// If prevMaxPartSize is set use that.
|
||||
if prevMaxPartSize != 0 {
|
||||
partSize = prevMaxPartSize
|
||||
}
|
||||
|
||||
// MD5 and SHA256 hasher.
|
||||
var hashMD5, hashSHA256 hash.Hash
|
||||
|
||||
// Part number always starts with prevPartNumber + 1. i.e The next part number.
|
||||
partNumber := prevPartNumber + 1
|
||||
|
||||
// Upload each part until totalUploadedSize reaches input reader size.
|
||||
for totalUploadedSize < size {
|
||||
// Initialize a new temporary file.
|
||||
tmpFile, err := newTempFile("multiparts$-putobject-partial")
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
// Create a hash multiwriter.
|
||||
hashMD5 = md5.New()
|
||||
hashWriter := io.MultiWriter(hashMD5)
|
||||
if c.signature.isV4() {
|
||||
hashSHA256 = sha256.New()
|
||||
hashWriter = io.MultiWriter(hashMD5, hashSHA256)
|
||||
}
|
||||
writer := io.MultiWriter(tmpFile, hashWriter)
|
||||
|
||||
// Choose totalUploadedSize as the current readAtOffset.
|
||||
readAtOffset := totalUploadedSize
|
||||
|
||||
// Read until partSize.
|
||||
var totalReadPartSize int64
|
||||
|
||||
// ReadAt defaults to reading at 5MiB buffer.
|
||||
readAtBuffer := make([]byte, optimalReadAtBufferSize)
|
||||
|
||||
// Following block reads data at an offset from the input
|
||||
// reader and copies data to into local temporary file.
|
||||
// Temporary file data is limited to the partSize.
|
||||
for totalReadPartSize < partSize {
|
||||
readAtSize, rerr := reader.ReadAt(readAtBuffer, readAtOffset)
|
||||
if rerr != nil {
|
||||
if rerr != io.EOF {
|
||||
return 0, rerr
|
||||
}
|
||||
}
|
||||
writeSize, werr := writer.Write(readAtBuffer[:readAtSize])
|
||||
if werr != nil {
|
||||
return 0, werr
|
||||
}
|
||||
if readAtSize != writeSize {
|
||||
return 0, errors.New("Something really bad happened here. " + reportIssue)
|
||||
}
|
||||
readAtOffset += int64(writeSize)
|
||||
totalReadPartSize += int64(writeSize)
|
||||
if rerr == io.EOF {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
// Seek back to beginning of the temporary file.
|
||||
if _, err := tmpFile.Seek(0, 0); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
var md5Sum, sha256Sum []byte
|
||||
md5Sum = hashMD5.Sum(nil)
|
||||
// Signature version '4'.
|
||||
if c.signature.isV4() {
|
||||
sha256Sum = hashSHA256.Sum(nil)
|
||||
}
|
||||
|
||||
// Proceed to upload the part.
|
||||
objPart, err := c.uploadPart(bucketName, objectName, uploadID, tmpFile, partNumber, md5Sum, sha256Sum, totalReadPartSize)
|
||||
if err != nil {
|
||||
// Close the read closer.
|
||||
tmpFile.Close()
|
||||
return totalUploadedSize, err
|
||||
}
|
||||
|
||||
// Save successfully uploaded size.
|
||||
totalUploadedSize += totalReadPartSize
|
||||
|
||||
// Save successfully uploaded part metadata.
|
||||
partsInfo[partNumber] = objPart
|
||||
|
||||
// Move to next part.
|
||||
partNumber++
|
||||
}
|
||||
|
||||
// Verify if we uploaded all the data.
|
||||
if totalUploadedSize != size {
|
||||
return totalUploadedSize, ErrUnexpectedEOF(totalUploadedSize, size, bucketName, objectName)
|
||||
}
|
||||
|
||||
// Loop over uploaded parts to save them in a Parts array before completing the multipart request.
|
||||
for _, part := range partsInfo {
|
||||
var complPart completePart
|
||||
complPart.ETag = part.ETag
|
||||
complPart.PartNumber = part.PartNumber
|
||||
completeMultipartUpload.Parts = append(completeMultipartUpload.Parts, complPart)
|
||||
}
|
||||
|
||||
// Sort all completed parts.
|
||||
sort.Sort(completedParts(completeMultipartUpload.Parts))
|
||||
_, err = c.completeMultipartUpload(bucketName, objectName, uploadID, completeMultipartUpload)
|
||||
if err != nil {
|
||||
return totalUploadedSize, err
|
||||
}
|
||||
|
||||
// Return final size.
|
||||
return totalUploadedSize, nil
|
||||
}
|
597
Godeps/_workspace/src/github.com/minio/minio-go/api-put-object.go
generated
vendored
597
Godeps/_workspace/src/github.com/minio/minio-go/api-put-object.go
generated
vendored
|
@ -18,21 +18,43 @@ package minio
|
|||
|
||||
import (
|
||||
"bytes"
|
||||
"crypto/md5"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/xml"
|
||||
"fmt"
|
||||
"hash"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"sort"
|
||||
"strconv"
|
||||
"os"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// getReaderSize gets the size of the underlying reader, if possible.
|
||||
func getReaderSize(reader io.Reader) (size int64, err error) {
|
||||
size = -1
|
||||
if reader != nil {
|
||||
switch v := reader.(type) {
|
||||
case *bytes.Buffer:
|
||||
size = int64(v.Len())
|
||||
case *bytes.Reader:
|
||||
size = int64(v.Len())
|
||||
case *strings.Reader:
|
||||
size = int64(v.Len())
|
||||
case *os.File:
|
||||
var st os.FileInfo
|
||||
st, err = v.Stat()
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
size = st.Size()
|
||||
case *Object:
|
||||
var st ObjectInfo
|
||||
st, err = v.Stat()
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
size = st.Size
|
||||
}
|
||||
}
|
||||
return size, nil
|
||||
}
|
||||
|
||||
// completedParts is a collection of parts sortable by their part numbers.
|
||||
// used for sorting the uploaded parts before completing the multipart request.
|
||||
type completedParts []completePart
|
||||
|
@ -54,7 +76,7 @@ func (a completedParts) Less(i, j int) bool { return a[i].PartNumber < a[j].Part
|
|||
// So we fall back to single PUT operation with the maximum limit of 5GiB.
|
||||
//
|
||||
// NOTE: For anonymous requests Amazon S3 doesn't allow multipart upload. So we fall back to single PUT operation.
|
||||
func (c Client) PutObject(bucketName, objectName string, data io.Reader, size int64, contentType string) (n int64, err error) {
|
||||
func (c Client) PutObject(bucketName, objectName string, reader io.Reader, contentType string) (n int64, err error) {
|
||||
// Input validation.
|
||||
if err := isValidBucketName(bucketName); err != nil {
|
||||
return 0, err
|
||||
|
@ -63,6 +85,17 @@ func (c Client) PutObject(bucketName, objectName string, data io.Reader, size in
|
|||
return 0, err
|
||||
}
|
||||
|
||||
// get reader size.
|
||||
size, err := getReaderSize(reader)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
// Check for largest object size allowed.
|
||||
if size > int64(maxMultipartPutObjectSize) {
|
||||
return 0, ErrEntityTooLarge(size, bucketName, objectName)
|
||||
}
|
||||
|
||||
// NOTE: Google Cloud Storage does not implement Amazon S3 Compatible multipart PUT.
|
||||
// So we fall back to single PUT operation with the maximum limit of 5GiB.
|
||||
if isGoogleEndpoint(c.endpointURL) {
|
||||
|
@ -74,55 +107,108 @@ func (c Client) PutObject(bucketName, objectName string, data io.Reader, size in
|
|||
BucketName: bucketName,
|
||||
}
|
||||
}
|
||||
if size > maxSinglePutObjectSize {
|
||||
return 0, ErrEntityTooLarge(size, bucketName, objectName)
|
||||
}
|
||||
// Do not compute MD5 for Google Cloud Storage. Uploads upto 5GiB in size.
|
||||
return c.putNoChecksum(bucketName, objectName, data, size, contentType)
|
||||
return c.putObjectNoChecksum(bucketName, objectName, reader, size, contentType)
|
||||
}
|
||||
|
||||
// NOTE: S3 doesn't allow anonymous multipart requests.
|
||||
if isAmazonEndpoint(c.endpointURL) && c.anonymous {
|
||||
if size <= -1 || size > int64(maxSinglePutObjectSize) {
|
||||
if size <= -1 {
|
||||
return 0, ErrorResponse{
|
||||
Code: "NotImplemented",
|
||||
Message: fmt.Sprintf("For anonymous requests Content-Length cannot be %d.", size),
|
||||
Message: "Content-Length cannot be negative for anonymous requests.",
|
||||
Key: objectName,
|
||||
BucketName: bucketName,
|
||||
}
|
||||
}
|
||||
// Do not compute MD5 for anonymous requests to Amazon S3. Uploads upto 5GiB in size.
|
||||
return c.putAnonymous(bucketName, objectName, data, size, contentType)
|
||||
}
|
||||
|
||||
// Large file upload is initiated for uploads for input data size
|
||||
// if its greater than 5MiB or data size is negative.
|
||||
if size >= minimumPartSize || size < 0 {
|
||||
return c.putLargeObject(bucketName, objectName, data, size, contentType)
|
||||
}
|
||||
return c.putSmallObject(bucketName, objectName, data, size, contentType)
|
||||
}
|
||||
|
||||
// putNoChecksum special function used Google Cloud Storage. This special function
|
||||
// is used for Google Cloud Storage since Google's multipart API is not S3 compatible.
|
||||
func (c Client) putNoChecksum(bucketName, objectName string, data io.Reader, size int64, contentType string) (n int64, err error) {
|
||||
// Input validation.
|
||||
if err := isValidBucketName(bucketName); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if err := isValidObjectName(objectName); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if size > maxPartSize {
|
||||
if size > maxSinglePutObjectSize {
|
||||
return 0, ErrEntityTooLarge(size, bucketName, objectName)
|
||||
}
|
||||
// For anonymous requests, we will not calculate sha256 and md5sum.
|
||||
putObjData := putObjectData{
|
||||
MD5Sum: nil,
|
||||
Sha256Sum: nil,
|
||||
ReadCloser: ioutil.NopCloser(data),
|
||||
Size: size,
|
||||
ContentType: contentType,
|
||||
// Do not compute MD5 for anonymous requests to Amazon S3. Uploads upto 5GiB in size.
|
||||
return c.putObjectNoChecksum(bucketName, objectName, reader, size, contentType)
|
||||
}
|
||||
|
||||
// putSmall object.
|
||||
if size < minimumPartSize && size > 0 {
|
||||
return c.putObjectSingle(bucketName, objectName, reader, size, contentType)
|
||||
}
|
||||
// For all sizes greater than 5MiB do multipart.
|
||||
n, err = c.putObjectMultipart(bucketName, objectName, reader, size, contentType)
|
||||
if err != nil {
|
||||
errResp := ToErrorResponse(err)
|
||||
// Verify if multipart functionality is not available, if not
|
||||
// fall back to single PutObject operation.
|
||||
if errResp.Code == "NotImplemented" {
|
||||
// Verify if size of reader is greater than '5GiB'.
|
||||
if size > maxSinglePutObjectSize {
|
||||
return 0, ErrEntityTooLarge(size, bucketName, objectName)
|
||||
}
|
||||
// Fall back to uploading as single PutObject operation.
|
||||
return c.putObjectSingle(bucketName, objectName, reader, size, contentType)
|
||||
}
|
||||
return n, err
|
||||
}
|
||||
return n, nil
|
||||
}
|
||||
|
||||
// putObjectNoChecksum special function used Google Cloud Storage. This special function
|
||||
// is used for Google Cloud Storage since Google's multipart API is not S3 compatible.
|
||||
func (c Client) putObjectNoChecksum(bucketName, objectName string, reader io.Reader, size int64, contentType string) (n int64, err error) {
|
||||
// Input validation.
|
||||
if err := isValidBucketName(bucketName); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if err := isValidObjectName(objectName); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if size > maxSinglePutObjectSize {
|
||||
return 0, ErrEntityTooLarge(size, bucketName, objectName)
|
||||
}
|
||||
// This function does not calculate sha256 and md5sum for payload.
|
||||
// Execute put object.
|
||||
st, err := c.putObjectDo(bucketName, objectName, ioutil.NopCloser(reader), nil, nil, size, contentType)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if st.Size != size {
|
||||
return 0, ErrUnexpectedEOF(st.Size, size, bucketName, objectName)
|
||||
}
|
||||
return size, nil
|
||||
}
|
||||
|
||||
// putObjectSingle is a special function for uploading single put object request.
|
||||
// This special function is used as a fallback when multipart upload fails.
|
||||
func (c Client) putObjectSingle(bucketName, objectName string, reader io.Reader, size int64, contentType string) (n int64, err error) {
|
||||
// Input validation.
|
||||
if err := isValidBucketName(bucketName); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if err := isValidObjectName(objectName); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if size > maxSinglePutObjectSize {
|
||||
return 0, ErrEntityTooLarge(size, bucketName, objectName)
|
||||
}
|
||||
// If size is a stream, upload upto 5GiB.
|
||||
if size <= -1 {
|
||||
size = maxSinglePutObjectSize
|
||||
}
|
||||
// Initialize a new temporary file.
|
||||
tmpFile, err := newTempFile("single$-putobject-single")
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
md5Sum, sha256Sum, size, err := c.hashCopyN(tmpFile, reader, size)
|
||||
if err != nil {
|
||||
if err != io.EOF {
|
||||
return 0, err
|
||||
}
|
||||
}
|
||||
// Execute put object.
|
||||
st, err := c.putObject(bucketName, objectName, putObjData)
|
||||
st, err := c.putObjectDo(bucketName, objectName, tmpFile, md5Sum, sha256Sum, size, contentType)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
@ -132,442 +218,67 @@ func (c Client) putNoChecksum(bucketName, objectName string, data io.Reader, siz
|
|||
return size, nil
|
||||
}
|
||||
|
||||
// putAnonymous is a special function for uploading content as anonymous request.
|
||||
// This special function is necessary since Amazon S3 doesn't allow anonymous
|
||||
// multipart uploads.
|
||||
func (c Client) putAnonymous(bucketName, objectName string, data io.Reader, size int64, contentType string) (n int64, err error) {
|
||||
// Input validation.
|
||||
if err := isValidBucketName(bucketName); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if err := isValidObjectName(objectName); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return c.putNoChecksum(bucketName, objectName, data, size, contentType)
|
||||
}
|
||||
|
||||
// putSmallObject uploads files smaller than 5 mega bytes.
|
||||
func (c Client) putSmallObject(bucketName, objectName string, data io.Reader, size int64, contentType string) (n int64, err error) {
|
||||
// Input validation.
|
||||
if err := isValidBucketName(bucketName); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if err := isValidObjectName(objectName); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
// Read input data fully into buffer.
|
||||
dataBytes, err := ioutil.ReadAll(data)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if int64(len(dataBytes)) != size {
|
||||
return 0, ErrUnexpectedEOF(int64(len(dataBytes)), size, bucketName, objectName)
|
||||
}
|
||||
// Construct a new PUT object metadata.
|
||||
putObjData := putObjectData{
|
||||
MD5Sum: sumMD5(dataBytes),
|
||||
Sha256Sum: sum256(dataBytes),
|
||||
ReadCloser: ioutil.NopCloser(bytes.NewReader(dataBytes)),
|
||||
Size: size,
|
||||
ContentType: contentType,
|
||||
}
|
||||
// Single part use case, use putObject directly.
|
||||
st, err := c.putObject(bucketName, objectName, putObjData)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if st.Size != size {
|
||||
return 0, ErrUnexpectedEOF(st.Size, size, bucketName, objectName)
|
||||
}
|
||||
return size, nil
|
||||
}
|
||||
|
||||
// hashCopy - calculates Md5sum and Sha256sum for upto partSize amount of bytes.
|
||||
func (c Client) hashCopy(writer io.ReadWriteSeeker, data io.Reader, partSize int64) (md5Sum, sha256Sum []byte, size int64, err error) {
|
||||
// MD5 and Sha256 hasher.
|
||||
var hashMD5, hashSha256 hash.Hash
|
||||
// MD5 and Sha256 hasher.
|
||||
hashMD5 = md5.New()
|
||||
hashWriter := io.MultiWriter(writer, hashMD5)
|
||||
if c.signature.isV4() {
|
||||
hashSha256 = sha256.New()
|
||||
hashWriter = io.MultiWriter(writer, hashMD5, hashSha256)
|
||||
}
|
||||
|
||||
// Copies to input at writer.
|
||||
size, err = io.CopyN(hashWriter, data, partSize)
|
||||
if err != nil {
|
||||
// If not EOF return error right here.
|
||||
if err != io.EOF {
|
||||
return nil, nil, 0, err
|
||||
}
|
||||
}
|
||||
|
||||
// Seek back to beginning of input, any error fail right here.
|
||||
if _, err := writer.Seek(0, 0); err != nil {
|
||||
return nil, nil, 0, err
|
||||
}
|
||||
|
||||
// Finalize md5shum and sha256 sum.
|
||||
md5Sum = hashMD5.Sum(nil)
|
||||
if c.signature.isV4() {
|
||||
sha256Sum = hashSha256.Sum(nil)
|
||||
}
|
||||
return md5Sum, sha256Sum, size, err
|
||||
}
|
||||
|
||||
// putLargeObject uploads files bigger than 5 mega bytes.
|
||||
func (c Client) putLargeObject(bucketName, objectName string, data io.Reader, size int64, contentType string) (n int64, err error) {
|
||||
// Input validation.
|
||||
if err := isValidBucketName(bucketName); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if err := isValidObjectName(objectName); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
// getUploadID for an object, initiates a new multipart request
|
||||
// if it cannot find any previously partially uploaded object.
|
||||
uploadID, err := c.getUploadID(bucketName, objectName, contentType)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
// total data read and written to server. should be equal to 'size' at the end of the call.
|
||||
var totalUploadedSize int64
|
||||
|
||||
// Complete multipart upload.
|
||||
var completeMultipartUpload completeMultipartUpload
|
||||
|
||||
// Fetch previously upload parts and save the total size.
|
||||
partsInfo, err := c.listObjectParts(bucketName, objectName, uploadID)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
// Previous maximum part size
|
||||
var prevMaxPartSize int64
|
||||
// Loop through all parts and calculate totalUploadedSize.
|
||||
for _, partInfo := range partsInfo {
|
||||
// Choose the maximum part size.
|
||||
if partInfo.Size >= prevMaxPartSize {
|
||||
prevMaxPartSize = partInfo.Size
|
||||
}
|
||||
}
|
||||
|
||||
// Calculate the optimal part size for a given size.
|
||||
partSize := optimalPartSize(size)
|
||||
// If prevMaxPartSize is set use that.
|
||||
if prevMaxPartSize != 0 {
|
||||
partSize = prevMaxPartSize
|
||||
}
|
||||
|
||||
// Part number always starts with '0'.
|
||||
partNumber := 0
|
||||
|
||||
// Loop through until EOF.
|
||||
for {
|
||||
// Increment part number.
|
||||
partNumber++
|
||||
|
||||
// Initialize a new temporary file.
|
||||
tmpFile, err := newTempFile("multiparts$-putobject")
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
// Calculates MD5 and Sha256 sum while copying partSize bytes into tmpFile.
|
||||
md5Sum, sha256Sum, size, rErr := c.hashCopy(tmpFile, data, partSize)
|
||||
if rErr != nil {
|
||||
if rErr != io.EOF {
|
||||
return 0, rErr
|
||||
}
|
||||
}
|
||||
|
||||
// Save all the part metadata.
|
||||
prtData := partData{
|
||||
ReadCloser: tmpFile,
|
||||
Size: size,
|
||||
MD5Sum: md5Sum,
|
||||
Sha256Sum: sha256Sum,
|
||||
Number: partNumber, // Current part number to be uploaded.
|
||||
}
|
||||
|
||||
// If part not uploaded proceed to upload.
|
||||
if !isPartUploaded(objectPart{
|
||||
ETag: hex.EncodeToString(prtData.MD5Sum),
|
||||
PartNumber: partNumber,
|
||||
}, partsInfo) {
|
||||
// execute upload part.
|
||||
objPart, err := c.uploadPart(bucketName, objectName, uploadID, prtData)
|
||||
if err != nil {
|
||||
// Close the read closer.
|
||||
prtData.ReadCloser.Close()
|
||||
return 0, err
|
||||
}
|
||||
// Save successfully uploaded part metadata.
|
||||
partsInfo[prtData.Number] = objPart
|
||||
}
|
||||
|
||||
// Close the read closer.
|
||||
prtData.ReadCloser.Close()
|
||||
|
||||
// If read error was an EOF, break out of the loop.
|
||||
if rErr == io.EOF {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
// Loop over uploaded parts to save them in a Parts array before completing the multipart request.
|
||||
for _, part := range partsInfo {
|
||||
var complPart completePart
|
||||
complPart.ETag = part.ETag
|
||||
complPart.PartNumber = part.PartNumber
|
||||
completeMultipartUpload.Parts = append(completeMultipartUpload.Parts, complPart)
|
||||
// Save successfully uploaded size.
|
||||
totalUploadedSize += part.Size
|
||||
}
|
||||
|
||||
// If size is greater than zero verify totalUploadedSize. if totalUploadedSize is
|
||||
// different than the input 'size', do not complete the request throw an error.
|
||||
if size > 0 {
|
||||
if totalUploadedSize != size {
|
||||
return totalUploadedSize, ErrUnexpectedEOF(totalUploadedSize, size, bucketName, objectName)
|
||||
}
|
||||
}
|
||||
|
||||
// If partNumber is different than total list of parts, error out.
|
||||
if partNumber != len(completeMultipartUpload.Parts) {
|
||||
return totalUploadedSize, ErrInvalidParts(partNumber, len(completeMultipartUpload.Parts))
|
||||
}
|
||||
|
||||
// Sort all completed parts.
|
||||
sort.Sort(completedParts(completeMultipartUpload.Parts))
|
||||
_, err = c.completeMultipartUpload(bucketName, objectName, uploadID, completeMultipartUpload)
|
||||
if err != nil {
|
||||
return totalUploadedSize, err
|
||||
}
|
||||
|
||||
// Return final size.
|
||||
return totalUploadedSize, nil
|
||||
}
|
||||
|
||||
// putObject - add an object to a bucket.
|
||||
// putObjectDo - executes the put object http operation.
|
||||
// NOTE: You must have WRITE permissions on a bucket to add an object to it.
|
||||
func (c Client) putObject(bucketName, objectName string, putObjData putObjectData) (ObjectStat, error) {
|
||||
func (c Client) putObjectDo(bucketName, objectName string, reader io.ReadCloser, md5Sum []byte, sha256Sum []byte, size int64, contentType string) (ObjectInfo, error) {
|
||||
// Input validation.
|
||||
if err := isValidBucketName(bucketName); err != nil {
|
||||
return ObjectStat{}, err
|
||||
return ObjectInfo{}, err
|
||||
}
|
||||
if err := isValidObjectName(objectName); err != nil {
|
||||
return ObjectStat{}, err
|
||||
return ObjectInfo{}, err
|
||||
}
|
||||
|
||||
if strings.TrimSpace(putObjData.ContentType) == "" {
|
||||
putObjData.ContentType = "application/octet-stream"
|
||||
if size <= -1 {
|
||||
return ObjectInfo{}, ErrEntityTooSmall(size, bucketName, objectName)
|
||||
}
|
||||
|
||||
if size > maxSinglePutObjectSize {
|
||||
return ObjectInfo{}, ErrEntityTooLarge(size, bucketName, objectName)
|
||||
}
|
||||
|
||||
if strings.TrimSpace(contentType) == "" {
|
||||
contentType = "application/octet-stream"
|
||||
}
|
||||
|
||||
// Set headers.
|
||||
customHeader := make(http.Header)
|
||||
customHeader.Set("Content-Type", putObjData.ContentType)
|
||||
customHeader.Set("Content-Type", contentType)
|
||||
|
||||
// Populate request metadata.
|
||||
reqMetadata := requestMetadata{
|
||||
bucketName: bucketName,
|
||||
objectName: objectName,
|
||||
customHeader: customHeader,
|
||||
contentBody: putObjData.ReadCloser,
|
||||
contentLength: putObjData.Size,
|
||||
contentSha256Bytes: putObjData.Sha256Sum,
|
||||
contentMD5Bytes: putObjData.MD5Sum,
|
||||
contentBody: reader,
|
||||
contentLength: size,
|
||||
contentMD5Bytes: md5Sum,
|
||||
contentSHA256Bytes: sha256Sum,
|
||||
}
|
||||
// Initiate new request.
|
||||
req, err := c.newRequest("PUT", reqMetadata)
|
||||
if err != nil {
|
||||
return ObjectStat{}, err
|
||||
return ObjectInfo{}, err
|
||||
}
|
||||
// Execute the request.
|
||||
resp, err := c.do(req)
|
||||
defer closeResponse(resp)
|
||||
if err != nil {
|
||||
return ObjectStat{}, err
|
||||
return ObjectInfo{}, err
|
||||
}
|
||||
if resp != nil {
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return ObjectStat{}, HTTPRespToErrorResponse(resp, bucketName, objectName)
|
||||
return ObjectInfo{}, HTTPRespToErrorResponse(resp, bucketName, objectName)
|
||||
}
|
||||
}
|
||||
|
||||
var metadata ObjectStat
|
||||
var metadata ObjectInfo
|
||||
// Trim off the odd double quotes from ETag in the beginning and end.
|
||||
metadata.ETag = strings.TrimPrefix(resp.Header.Get("ETag"), "\"")
|
||||
metadata.ETag = strings.TrimSuffix(metadata.ETag, "\"")
|
||||
// A success here means data was written to server successfully.
|
||||
metadata.Size = putObjData.Size
|
||||
metadata.Size = size
|
||||
|
||||
// Return here.
|
||||
return metadata, nil
|
||||
}
|
||||
|
||||
// initiateMultipartUpload initiates a multipart upload and returns an upload ID.
|
||||
func (c Client) initiateMultipartUpload(bucketName, objectName, contentType string) (initiateMultipartUploadResult, error) {
|
||||
// Input validation.
|
||||
if err := isValidBucketName(bucketName); err != nil {
|
||||
return initiateMultipartUploadResult{}, err
|
||||
}
|
||||
if err := isValidObjectName(objectName); err != nil {
|
||||
return initiateMultipartUploadResult{}, err
|
||||
}
|
||||
|
||||
// Initialize url queries.
|
||||
urlValues := make(url.Values)
|
||||
urlValues.Set("uploads", "")
|
||||
|
||||
if contentType == "" {
|
||||
contentType = "application/octet-stream"
|
||||
}
|
||||
|
||||
// set ContentType header.
|
||||
customHeader := make(http.Header)
|
||||
customHeader.Set("Content-Type", contentType)
|
||||
|
||||
reqMetadata := requestMetadata{
|
||||
bucketName: bucketName,
|
||||
objectName: objectName,
|
||||
queryValues: urlValues,
|
||||
customHeader: customHeader,
|
||||
}
|
||||
|
||||
// Instantiate the request.
|
||||
req, err := c.newRequest("POST", reqMetadata)
|
||||
if err != nil {
|
||||
return initiateMultipartUploadResult{}, err
|
||||
}
|
||||
// Execute the request.
|
||||
resp, err := c.do(req)
|
||||
defer closeResponse(resp)
|
||||
if err != nil {
|
||||
return initiateMultipartUploadResult{}, err
|
||||
}
|
||||
if resp != nil {
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return initiateMultipartUploadResult{}, HTTPRespToErrorResponse(resp, bucketName, objectName)
|
||||
}
|
||||
}
|
||||
// Decode xml initiate multipart.
|
||||
initiateMultipartUploadResult := initiateMultipartUploadResult{}
|
||||
err = xmlDecoder(resp.Body, &initiateMultipartUploadResult)
|
||||
if err != nil {
|
||||
return initiateMultipartUploadResult, err
|
||||
}
|
||||
return initiateMultipartUploadResult, nil
|
||||
}
|
||||
|
||||
// uploadPart uploads a part in a multipart upload.
|
||||
func (c Client) uploadPart(bucketName, objectName, uploadID string, uploadingPart partData) (objectPart, error) {
|
||||
// Input validation.
|
||||
if err := isValidBucketName(bucketName); err != nil {
|
||||
return objectPart{}, err
|
||||
}
|
||||
if err := isValidObjectName(objectName); err != nil {
|
||||
return objectPart{}, err
|
||||
}
|
||||
|
||||
// Get resources properly escaped and lined up before using them in http request.
|
||||
urlValues := make(url.Values)
|
||||
// Set part number.
|
||||
urlValues.Set("partNumber", strconv.Itoa(uploadingPart.Number))
|
||||
// Set upload id.
|
||||
urlValues.Set("uploadId", uploadID)
|
||||
|
||||
reqMetadata := requestMetadata{
|
||||
bucketName: bucketName,
|
||||
objectName: objectName,
|
||||
queryValues: urlValues,
|
||||
contentBody: uploadingPart.ReadCloser,
|
||||
contentLength: uploadingPart.Size,
|
||||
contentSha256Bytes: uploadingPart.Sha256Sum,
|
||||
contentMD5Bytes: uploadingPart.MD5Sum,
|
||||
}
|
||||
|
||||
// Instantiate a request.
|
||||
req, err := c.newRequest("PUT", reqMetadata)
|
||||
if err != nil {
|
||||
return objectPart{}, err
|
||||
}
|
||||
// Execute the request.
|
||||
resp, err := c.do(req)
|
||||
defer closeResponse(resp)
|
||||
if err != nil {
|
||||
return objectPart{}, err
|
||||
}
|
||||
if resp != nil {
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return objectPart{}, HTTPRespToErrorResponse(resp, bucketName, objectName)
|
||||
}
|
||||
}
|
||||
// Once successfully uploaded, return completed part.
|
||||
objPart := objectPart{}
|
||||
objPart.Size = uploadingPart.Size
|
||||
objPart.PartNumber = uploadingPart.Number
|
||||
// Trim off the odd double quotes from ETag in the beginning and end.
|
||||
objPart.ETag = strings.TrimPrefix(resp.Header.Get("ETag"), "\"")
|
||||
objPart.ETag = strings.TrimSuffix(objPart.ETag, "\"")
|
||||
return objPart, nil
|
||||
}
|
||||
|
||||
// completeMultipartUpload completes a multipart upload by assembling previously uploaded parts.
|
||||
func (c Client) completeMultipartUpload(bucketName, objectName, uploadID string, complete completeMultipartUpload) (completeMultipartUploadResult, error) {
|
||||
// Input validation.
|
||||
if err := isValidBucketName(bucketName); err != nil {
|
||||
return completeMultipartUploadResult{}, err
|
||||
}
|
||||
if err := isValidObjectName(objectName); err != nil {
|
||||
return completeMultipartUploadResult{}, err
|
||||
}
|
||||
|
||||
// Initialize url queries.
|
||||
urlValues := make(url.Values)
|
||||
urlValues.Set("uploadId", uploadID)
|
||||
|
||||
// Marshal complete multipart body.
|
||||
completeMultipartUploadBytes, err := xml.Marshal(complete)
|
||||
if err != nil {
|
||||
return completeMultipartUploadResult{}, err
|
||||
}
|
||||
|
||||
// Instantiate all the complete multipart buffer.
|
||||
completeMultipartUploadBuffer := bytes.NewBuffer(completeMultipartUploadBytes)
|
||||
reqMetadata := requestMetadata{
|
||||
bucketName: bucketName,
|
||||
objectName: objectName,
|
||||
queryValues: urlValues,
|
||||
contentBody: ioutil.NopCloser(completeMultipartUploadBuffer),
|
||||
contentLength: int64(completeMultipartUploadBuffer.Len()),
|
||||
contentSha256Bytes: sum256(completeMultipartUploadBuffer.Bytes()),
|
||||
}
|
||||
|
||||
// Instantiate the request.
|
||||
req, err := c.newRequest("POST", reqMetadata)
|
||||
if err != nil {
|
||||
return completeMultipartUploadResult{}, err
|
||||
}
|
||||
|
||||
// Execute the request.
|
||||
resp, err := c.do(req)
|
||||
defer closeResponse(resp)
|
||||
if err != nil {
|
||||
return completeMultipartUploadResult{}, err
|
||||
}
|
||||
if resp != nil {
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return completeMultipartUploadResult{}, HTTPRespToErrorResponse(resp, bucketName, objectName)
|
||||
}
|
||||
}
|
||||
// If successful response, decode the body.
|
||||
completeMultipartUploadResult := completeMultipartUploadResult{}
|
||||
err = xmlDecoder(resp.Body, &completeMultipartUploadResult)
|
||||
if err != nil {
|
||||
return completeMultipartUploadResult, err
|
||||
}
|
||||
return completeMultipartUploadResult, nil
|
||||
}
|
||||
|
|
39
Godeps/_workspace/src/github.com/minio/minio-go/api-remove.go
generated
vendored
39
Godeps/_workspace/src/github.com/minio/minio-go/api-remove.go
generated
vendored
|
@ -26,15 +26,18 @@ import (
|
|||
// All objects (including all object versions and delete markers).
|
||||
// in the bucket must be deleted before successfully attempting this request.
|
||||
func (c Client) RemoveBucket(bucketName string) error {
|
||||
// Input validation.
|
||||
if err := isValidBucketName(bucketName); err != nil {
|
||||
return err
|
||||
}
|
||||
// Instantiate a new request.
|
||||
req, err := c.newRequest("DELETE", requestMetadata{
|
||||
bucketName: bucketName,
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// Initiate the request.
|
||||
resp, err := c.do(req)
|
||||
defer closeResponse(resp)
|
||||
if err != nil {
|
||||
|
@ -54,12 +57,14 @@ func (c Client) RemoveBucket(bucketName string) error {
|
|||
|
||||
// RemoveObject remove an object from a bucket.
|
||||
func (c Client) RemoveObject(bucketName, objectName string) error {
|
||||
// Input validation.
|
||||
if err := isValidBucketName(bucketName); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := isValidObjectName(objectName); err != nil {
|
||||
return err
|
||||
}
|
||||
// Instantiate the request.
|
||||
req, err := c.newRequest("DELETE", requestMetadata{
|
||||
bucketName: bucketName,
|
||||
objectName: objectName,
|
||||
|
@ -67,6 +72,7 @@ func (c Client) RemoveObject(bucketName, objectName string) error {
|
|||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// Initiate the request.
|
||||
resp, err := c.do(req)
|
||||
defer closeResponse(resp)
|
||||
if err != nil {
|
||||
|
@ -81,42 +87,32 @@ func (c Client) RemoveObject(bucketName, objectName string) error {
|
|||
// RemoveIncompleteUpload aborts an partially uploaded object.
|
||||
// Requires explicit authentication, no anonymous requests are allowed for multipart API.
|
||||
func (c Client) RemoveIncompleteUpload(bucketName, objectName string) error {
|
||||
// Validate input arguments.
|
||||
// Input validation.
|
||||
if err := isValidBucketName(bucketName); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := isValidObjectName(objectName); err != nil {
|
||||
return err
|
||||
}
|
||||
errorCh := make(chan error)
|
||||
go func(errorCh chan<- error) {
|
||||
defer close(errorCh)
|
||||
// Find multipart upload id of the object.
|
||||
// Find multipart upload id of the object to be aborted.
|
||||
uploadID, err := c.findUploadID(bucketName, objectName)
|
||||
if err != nil {
|
||||
errorCh <- err
|
||||
return
|
||||
return err
|
||||
}
|
||||
if uploadID != "" {
|
||||
// If uploadID is not an empty string, initiate the request.
|
||||
// Upload id found, abort the incomplete multipart upload.
|
||||
err := c.abortMultipartUpload(bucketName, objectName, uploadID)
|
||||
if err != nil {
|
||||
errorCh <- err
|
||||
return
|
||||
}
|
||||
return
|
||||
}
|
||||
}(errorCh)
|
||||
err, ok := <-errorCh
|
||||
if ok && err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// abortMultipartUpload aborts a multipart upload for the given uploadID, all parts are deleted.
|
||||
// abortMultipartUpload aborts a multipart upload for the given
|
||||
// uploadID, all previously uploaded parts are deleted.
|
||||
func (c Client) abortMultipartUpload(bucketName, objectName, uploadID string) error {
|
||||
// Validate input arguments.
|
||||
// Input validation.
|
||||
if err := isValidBucketName(bucketName); err != nil {
|
||||
return err
|
||||
}
|
||||
|
@ -138,7 +134,7 @@ func (c Client) abortMultipartUpload(bucketName, objectName, uploadID string) er
|
|||
return err
|
||||
}
|
||||
|
||||
// execute the request.
|
||||
// Initiate the request.
|
||||
resp, err := c.do(req)
|
||||
defer closeResponse(resp)
|
||||
if err != nil {
|
||||
|
@ -146,11 +142,12 @@ func (c Client) abortMultipartUpload(bucketName, objectName, uploadID string) er
|
|||
}
|
||||
if resp != nil {
|
||||
if resp.StatusCode != http.StatusNoContent {
|
||||
// Abort has no response body, handle it.
|
||||
// Abort has no response body, handle it for any errors.
|
||||
var errorResponse ErrorResponse
|
||||
switch resp.StatusCode {
|
||||
case http.StatusNotFound:
|
||||
// This is needed specifically for Abort and it cannot be converged.
|
||||
// This is needed specifically for abort and it cannot
|
||||
// be converged into default case.
|
||||
errorResponse = ErrorResponse{
|
||||
Code: "NoSuchUpload",
|
||||
Message: "The specified multipart upload does not exist.",
|
||||
|
|
64
Godeps/_workspace/src/github.com/minio/minio-go/api-s3-definitions.go
generated
vendored
64
Godeps/_workspace/src/github.com/minio/minio-go/api-s3-definitions.go
generated
vendored
|
@ -21,30 +21,33 @@ import (
|
|||
"time"
|
||||
)
|
||||
|
||||
// listAllMyBucketsResult container for listBuckets response
|
||||
// listAllMyBucketsResult container for listBuckets response.
|
||||
type listAllMyBucketsResult struct {
|
||||
// Container for one or more buckets.
|
||||
Buckets struct {
|
||||
Bucket []BucketStat
|
||||
Bucket []BucketInfo
|
||||
}
|
||||
Owner owner
|
||||
}
|
||||
|
||||
// owner container for bucket owner information
|
||||
// owner container for bucket owner information.
|
||||
type owner struct {
|
||||
DisplayName string
|
||||
ID string
|
||||
}
|
||||
|
||||
// commonPrefix container for prefix response
|
||||
// commonPrefix container for prefix response.
|
||||
type commonPrefix struct {
|
||||
Prefix string
|
||||
}
|
||||
|
||||
// listBucketResult container for listObjects response
|
||||
// listBucketResult container for listObjects response.
|
||||
type listBucketResult struct {
|
||||
CommonPrefixes []commonPrefix // A response can contain CommonPrefixes only if you have specified a delimiter
|
||||
Contents []ObjectStat // Metadata about each object returned
|
||||
// A response can contain CommonPrefixes only if you have
|
||||
// specified a delimiter.
|
||||
CommonPrefixes []commonPrefix
|
||||
// Metadata about each object returned.
|
||||
Contents []ObjectInfo
|
||||
Delimiter string
|
||||
|
||||
// Encoding type used to encode object keys in the response.
|
||||
|
@ -57,13 +60,15 @@ type listBucketResult struct {
|
|||
MaxKeys int64
|
||||
Name string
|
||||
|
||||
// When response is truncated (the IsTruncated element value in the response
|
||||
// is true), you can use the key name in this field as marker in the subsequent
|
||||
// request to get next set of objects. Object storage lists objects in alphabetical
|
||||
// order Note: This element is returned only if you have delimiter request parameter
|
||||
// specified. If response does not include the NextMaker and it is truncated,
|
||||
// you can use the value of the last Key in the response as the marker in the
|
||||
// subsequent request to get the next set of object keys.
|
||||
// When response is truncated (the IsTruncated element value in
|
||||
// the response is true), you can use the key name in this field
|
||||
// as marker in the subsequent request to get next set of objects.
|
||||
// Object storage lists objects in alphabetical order Note: This
|
||||
// element is returned only if you have delimiter request
|
||||
// parameter specified. If response does not include the NextMaker
|
||||
// and it is truncated, you can use the value of the last Key in
|
||||
// the response as the marker in the subsequent request to get the
|
||||
// next set of object keys.
|
||||
NextMarker string
|
||||
Prefix string
|
||||
}
|
||||
|
@ -78,19 +83,20 @@ type listMultipartUploadsResult struct {
|
|||
EncodingType string
|
||||
MaxUploads int64
|
||||
IsTruncated bool
|
||||
Uploads []ObjectMultipartStat `xml:"Upload"`
|
||||
Uploads []ObjectMultipartInfo `xml:"Upload"`
|
||||
Prefix string
|
||||
Delimiter string
|
||||
CommonPrefixes []commonPrefix // A response can contain CommonPrefixes only if you specify a delimiter
|
||||
// A response can contain CommonPrefixes only if you specify a delimiter.
|
||||
CommonPrefixes []commonPrefix
|
||||
}
|
||||
|
||||
// initiator container for who initiated multipart upload
|
||||
// initiator container for who initiated multipart upload.
|
||||
type initiator struct {
|
||||
ID string
|
||||
DisplayName string
|
||||
}
|
||||
|
||||
// objectPart container for particular part of an object
|
||||
// objectPart container for particular part of an object.
|
||||
type objectPart struct {
|
||||
// Part number identifies the part.
|
||||
PartNumber int
|
||||
|
@ -98,7 +104,8 @@ type objectPart struct {
|
|||
// Date and time the part was uploaded.
|
||||
LastModified time.Time
|
||||
|
||||
// Entity tag returned when the part was uploaded, usually md5sum of the part
|
||||
// Entity tag returned when the part was uploaded, usually md5sum
|
||||
// of the part.
|
||||
ETag string
|
||||
|
||||
// Size of the uploaded part data.
|
||||
|
@ -126,14 +133,16 @@ type listObjectPartsResult struct {
|
|||
EncodingType string
|
||||
}
|
||||
|
||||
// initiateMultipartUploadResult container for InitiateMultiPartUpload response.
|
||||
// initiateMultipartUploadResult container for InitiateMultiPartUpload
|
||||
// response.
|
||||
type initiateMultipartUploadResult struct {
|
||||
Bucket string
|
||||
Key string
|
||||
UploadID string `xml:"UploadId"`
|
||||
}
|
||||
|
||||
// completeMultipartUploadResult container for completed multipart upload response.
|
||||
// completeMultipartUploadResult container for completed multipart
|
||||
// upload response.
|
||||
type completeMultipartUploadResult struct {
|
||||
Location string
|
||||
Bucket string
|
||||
|
@ -141,7 +150,8 @@ type completeMultipartUploadResult struct {
|
|||
ETag string
|
||||
}
|
||||
|
||||
// completePart sub container lists individual part numbers and their md5sum, part of completeMultipartUpload.
|
||||
// completePart sub container lists individual part numbers and their
|
||||
// md5sum, part of completeMultipartUpload.
|
||||
type completePart struct {
|
||||
XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Part" json:"-"`
|
||||
|
||||
|
@ -150,13 +160,13 @@ type completePart struct {
|
|||
ETag string
|
||||
}
|
||||
|
||||
// completeMultipartUpload container for completing multipart upload
|
||||
// completeMultipartUpload container for completing multipart upload.
|
||||
type completeMultipartUpload struct {
|
||||
XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ CompleteMultipartUpload" json:"-"`
|
||||
Parts []completePart `xml:"Part"`
|
||||
}
|
||||
|
||||
// createBucketConfiguration container for bucket configuration
|
||||
// createBucketConfiguration container for bucket configuration.
|
||||
type createBucketConfiguration struct {
|
||||
XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ CreateBucketConfiguration" json:"-"`
|
||||
Location string `xml:"LocationConstraint"`
|
||||
|
@ -164,7 +174,8 @@ type createBucketConfiguration struct {
|
|||
|
||||
// grant container for the grantee and his or her permissions.
|
||||
type grant struct {
|
||||
// grantee container for DisplayName and ID of the person being granted permissions.
|
||||
// grantee container for DisplayName and ID of the person being
|
||||
// granted permissions.
|
||||
Grantee struct {
|
||||
ID string
|
||||
DisplayName string
|
||||
|
@ -175,7 +186,8 @@ type grant struct {
|
|||
Permission string
|
||||
}
|
||||
|
||||
// accessControlPolicy contains the elements providing ACL permissions for a bucket.
|
||||
// accessControlPolicy contains the elements providing ACL permissions
|
||||
// for a bucket.
|
||||
type accessControlPolicy struct {
|
||||
// accessControlList container for ACL information.
|
||||
AccessControlList struct {
|
||||
|
|
25
Godeps/_workspace/src/github.com/minio/minio-go/api-stat.go
generated
vendored
25
Godeps/_workspace/src/github.com/minio/minio-go/api-stat.go
generated
vendored
|
@ -25,15 +25,18 @@ import (
|
|||
|
||||
// BucketExists verify if bucket exists and you have permission to access it.
|
||||
func (c Client) BucketExists(bucketName string) error {
|
||||
// Input validation.
|
||||
if err := isValidBucketName(bucketName); err != nil {
|
||||
return err
|
||||
}
|
||||
// Instantiate a new request.
|
||||
req, err := c.newRequest("HEAD", requestMetadata{
|
||||
bucketName: bucketName,
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// Initiate the request.
|
||||
resp, err := c.do(req)
|
||||
defer closeResponse(resp)
|
||||
if err != nil {
|
||||
|
@ -48,12 +51,13 @@ func (c Client) BucketExists(bucketName string) error {
|
|||
}
|
||||
|
||||
// StatObject verifies if object exists and you have permission to access.
|
||||
func (c Client) StatObject(bucketName, objectName string) (ObjectStat, error) {
|
||||
func (c Client) StatObject(bucketName, objectName string) (ObjectInfo, error) {
|
||||
// Input validation.
|
||||
if err := isValidBucketName(bucketName); err != nil {
|
||||
return ObjectStat{}, err
|
||||
return ObjectInfo{}, err
|
||||
}
|
||||
if err := isValidObjectName(objectName); err != nil {
|
||||
return ObjectStat{}, err
|
||||
return ObjectInfo{}, err
|
||||
}
|
||||
// Instantiate a new request.
|
||||
req, err := c.newRequest("HEAD", requestMetadata{
|
||||
|
@ -61,16 +65,17 @@ func (c Client) StatObject(bucketName, objectName string) (ObjectStat, error) {
|
|||
objectName: objectName,
|
||||
})
|
||||
if err != nil {
|
||||
return ObjectStat{}, err
|
||||
return ObjectInfo{}, err
|
||||
}
|
||||
// Initiate the request.
|
||||
resp, err := c.do(req)
|
||||
defer closeResponse(resp)
|
||||
if err != nil {
|
||||
return ObjectStat{}, err
|
||||
return ObjectInfo{}, err
|
||||
}
|
||||
if resp != nil {
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return ObjectStat{}, HTTPRespToErrorResponse(resp, bucketName, objectName)
|
||||
return ObjectInfo{}, HTTPRespToErrorResponse(resp, bucketName, objectName)
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -81,7 +86,7 @@ func (c Client) StatObject(bucketName, objectName string) (ObjectStat, error) {
|
|||
// Parse content length.
|
||||
size, err := strconv.ParseInt(resp.Header.Get("Content-Length"), 10, 64)
|
||||
if err != nil {
|
||||
return ObjectStat{}, ErrorResponse{
|
||||
return ObjectInfo{}, ErrorResponse{
|
||||
Code: "InternalError",
|
||||
Message: "Content-Length is invalid. " + reportIssue,
|
||||
BucketName: bucketName,
|
||||
|
@ -91,9 +96,10 @@ func (c Client) StatObject(bucketName, objectName string) (ObjectStat, error) {
|
|||
AmzBucketRegion: resp.Header.Get("x-amz-bucket-region"),
|
||||
}
|
||||
}
|
||||
// Parse Last-Modified has http time format.
|
||||
date, err := time.Parse(http.TimeFormat, resp.Header.Get("Last-Modified"))
|
||||
if err != nil {
|
||||
return ObjectStat{}, ErrorResponse{
|
||||
return ObjectInfo{}, ErrorResponse{
|
||||
Code: "InternalError",
|
||||
Message: "Last-Modified time format is invalid. " + reportIssue,
|
||||
BucketName: bucketName,
|
||||
|
@ -103,12 +109,13 @@ func (c Client) StatObject(bucketName, objectName string) (ObjectStat, error) {
|
|||
AmzBucketRegion: resp.Header.Get("x-amz-bucket-region"),
|
||||
}
|
||||
}
|
||||
// Fetch content type if any present.
|
||||
contentType := strings.TrimSpace(resp.Header.Get("Content-Type"))
|
||||
if contentType == "" {
|
||||
contentType = "application/octet-stream"
|
||||
}
|
||||
// Save object metadata info.
|
||||
var objectStat ObjectStat
|
||||
var objectStat ObjectInfo
|
||||
objectStat.ETag = md5sum
|
||||
objectStat.Key = objectName
|
||||
objectStat.Size = size
|
||||
|
|
125
Godeps/_workspace/src/github.com/minio/minio-go/api.go
generated
vendored
125
Godeps/_workspace/src/github.com/minio/minio-go/api.go
generated
vendored
|
@ -25,6 +25,7 @@ import (
|
|||
"net/http/httputil"
|
||||
"net/url"
|
||||
"os"
|
||||
"regexp"
|
||||
"runtime"
|
||||
"strings"
|
||||
"time"
|
||||
|
@ -33,10 +34,15 @@ import (
|
|||
// Client implements Amazon S3 compatible methods.
|
||||
type Client struct {
|
||||
/// Standard options.
|
||||
accessKeyID string // AccessKeyID required for authorized requests.
|
||||
secretAccessKey string // SecretAccessKey required for authorized requests.
|
||||
signature SignatureType // Choose a signature type if necessary.
|
||||
anonymous bool // Set to 'true' if Client has no access and secret keys.
|
||||
|
||||
// AccessKeyID required for authorized requests.
|
||||
accessKeyID string
|
||||
// SecretAccessKey required for authorized requests.
|
||||
secretAccessKey string
|
||||
// Choose a signature type if necessary.
|
||||
signature SignatureType
|
||||
// Set to 'true' if Client has no access and secret keys.
|
||||
anonymous bool
|
||||
|
||||
// User supplied.
|
||||
appInfo struct {
|
||||
|
@ -69,7 +75,8 @@ const (
|
|||
libraryUserAgent = libraryUserAgentPrefix + libraryName + "/" + libraryVersion
|
||||
)
|
||||
|
||||
// NewV2 - instantiate minio client with Amazon S3 signature version '2' compatiblity.
|
||||
// NewV2 - instantiate minio client with Amazon S3 signature version
|
||||
// '2' compatiblity.
|
||||
func NewV2(endpoint string, accessKeyID, secretAccessKey string, insecure bool) (CloudStorageClient, error) {
|
||||
clnt, err := privateNew(endpoint, accessKeyID, secretAccessKey, insecure)
|
||||
if err != nil {
|
||||
|
@ -80,7 +87,8 @@ func NewV2(endpoint string, accessKeyID, secretAccessKey string, insecure bool)
|
|||
return clnt, nil
|
||||
}
|
||||
|
||||
// NewV4 - instantiate minio client with Amazon S3 signature version '4' compatibility.
|
||||
// NewV4 - instantiate minio client with Amazon S3 signature version
|
||||
// '4' compatibility.
|
||||
func NewV4(endpoint string, accessKeyID, secretAccessKey string, insecure bool) (CloudStorageClient, error) {
|
||||
clnt, err := privateNew(endpoint, accessKeyID, secretAccessKey, insecure)
|
||||
if err != nil {
|
||||
|
@ -91,13 +99,15 @@ func NewV4(endpoint string, accessKeyID, secretAccessKey string, insecure bool)
|
|||
return clnt, nil
|
||||
}
|
||||
|
||||
// New - instantiate minio client Client, adds automatic verification of signature.
|
||||
// New - instantiate minio client Client, adds automatic verification
|
||||
// of signature.
|
||||
func New(endpoint string, accessKeyID, secretAccessKey string, insecure bool) (CloudStorageClient, error) {
|
||||
clnt, err := privateNew(endpoint, accessKeyID, secretAccessKey, insecure)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// Google cloud storage should be set to signature V2, force it if not.
|
||||
// Google cloud storage should be set to signature V2, force it if
|
||||
// not.
|
||||
if isGoogleEndpoint(clnt.endpointURL) {
|
||||
clnt.signature = SignatureV2
|
||||
}
|
||||
|
@ -136,7 +146,8 @@ func privateNew(endpoint, accessKeyID, secretAccessKey string, insecure bool) (*
|
|||
|
||||
// SetAppInfo - add application details to user agent.
|
||||
func (c *Client) SetAppInfo(appName string, appVersion string) {
|
||||
// if app name and version is not set, we do not a new user agent.
|
||||
// if app name and version is not set, we do not a new user
|
||||
// agent.
|
||||
if appName != "" && appVersion != "" {
|
||||
c.appInfo = struct {
|
||||
appName string
|
||||
|
@ -149,12 +160,13 @@ func (c *Client) SetAppInfo(appName string, appVersion string) {
|
|||
|
||||
// SetCustomTransport - set new custom transport.
|
||||
func (c *Client) SetCustomTransport(customHTTPTransport http.RoundTripper) {
|
||||
// Set this to override default transport ``http.DefaultTransport``.
|
||||
// Set this to override default transport
|
||||
// ``http.DefaultTransport``.
|
||||
//
|
||||
// This transport is usually needed for debugging OR to add your own
|
||||
// custom TLS certificates on the client transport, for custom CA's and
|
||||
// certs which are not part of standard certificate authority follow this
|
||||
// example :-
|
||||
// This transport is usually needed for debugging OR to add your
|
||||
// own custom TLS certificates on the client transport, for custom
|
||||
// CA's and certs which are not part of standard certificate
|
||||
// authority follow this example :-
|
||||
//
|
||||
// tr := &http.Transport{
|
||||
// TLSClientConfig: &tls.Config{RootCAs: pool},
|
||||
|
@ -187,7 +199,8 @@ func (c *Client) TraceOff() {
|
|||
c.isTraceEnabled = false
|
||||
}
|
||||
|
||||
// requestMetadata - is container for all the values to make a request.
|
||||
// requestMetadata - is container for all the values to make a
|
||||
// request.
|
||||
type requestMetadata struct {
|
||||
// If set newRequest presigns the URL.
|
||||
presignURL bool
|
||||
|
@ -202,10 +215,41 @@ type requestMetadata struct {
|
|||
// Generated by our internal code.
|
||||
contentBody io.ReadCloser
|
||||
contentLength int64
|
||||
contentSha256Bytes []byte
|
||||
contentSHA256Bytes []byte
|
||||
contentMD5Bytes []byte
|
||||
}
|
||||
|
||||
// Filter out signature value from Authorization header.
|
||||
func (c Client) filterSignature(req *http.Request) {
|
||||
// For anonymous requests return here.
|
||||
if c.anonymous {
|
||||
return
|
||||
}
|
||||
// Handle if Signature V2.
|
||||
if c.signature.isV2() {
|
||||
// Set a temporary redacted auth
|
||||
req.Header.Set("Authorization", "AWS **REDACTED**:**REDACTED**")
|
||||
return
|
||||
}
|
||||
|
||||
/// Signature V4 authorization header.
|
||||
|
||||
// Save the original auth.
|
||||
origAuth := req.Header.Get("Authorization")
|
||||
// Strip out accessKeyID from:
|
||||
// Credential=<access-key-id>/<date>/<aws-region>/<aws-service>/aws4_request
|
||||
regCred := regexp.MustCompile("Credential=([A-Z0-9]+)/")
|
||||
newAuth := regCred.ReplaceAllString(origAuth, "Credential=**REDACTED**/")
|
||||
|
||||
// Strip out 256-bit signature from: Signature=<256-bit signature>
|
||||
regSign := regexp.MustCompile("Signature=([[0-9a-f]+)")
|
||||
newAuth = regSign.ReplaceAllString(newAuth, "Signature=**REDACTED**")
|
||||
|
||||
// Set a temporary redacted auth
|
||||
req.Header.Set("Authorization", newAuth)
|
||||
return
|
||||
}
|
||||
|
||||
// dumpHTTP - dump HTTP request and response.
|
||||
func (c Client) dumpHTTP(req *http.Request, resp *http.Response) error {
|
||||
// Starts http dump.
|
||||
|
@ -214,6 +258,9 @@ func (c Client) dumpHTTP(req *http.Request, resp *http.Response) error {
|
|||
return err
|
||||
}
|
||||
|
||||
// Filter out Signature field from Authorization header.
|
||||
c.filterSignature(req)
|
||||
|
||||
// Only display request header.
|
||||
reqTrace, err := httputil.DumpRequestOut(req, false)
|
||||
if err != nil {
|
||||
|
@ -227,11 +274,22 @@ func (c Client) dumpHTTP(req *http.Request, resp *http.Response) error {
|
|||
}
|
||||
|
||||
// Only display response header.
|
||||
respTrace, err := httputil.DumpResponse(resp, false)
|
||||
var respTrace []byte
|
||||
|
||||
// For errors we make sure to dump response body as well.
|
||||
if resp.StatusCode != http.StatusOK &&
|
||||
resp.StatusCode != http.StatusPartialContent &&
|
||||
resp.StatusCode != http.StatusNoContent {
|
||||
respTrace, err = httputil.DumpResponse(resp, true)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
} else {
|
||||
respTrace, err = httputil.DumpResponse(resp, false)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
// Write response to trace output.
|
||||
_, err = fmt.Fprint(c.traceOutput, strings.TrimSuffix(string(respTrace), "\r\n"))
|
||||
if err != nil {
|
||||
|
@ -328,11 +386,12 @@ func (c Client) newRequest(method string, metadata requestMetadata) (*http.Reque
|
|||
|
||||
// Set sha256 sum only for non anonymous credentials.
|
||||
if !c.anonymous {
|
||||
// set sha256 sum for signature calculation only with signature version '4'.
|
||||
// set sha256 sum for signature calculation only with
|
||||
// signature version '4'.
|
||||
if c.signature.isV4() {
|
||||
req.Header.Set("X-Amz-Content-Sha256", hex.EncodeToString(sum256([]byte{})))
|
||||
if metadata.contentSha256Bytes != nil {
|
||||
req.Header.Set("X-Amz-Content-Sha256", hex.EncodeToString(metadata.contentSha256Bytes))
|
||||
if metadata.contentSHA256Bytes != nil {
|
||||
req.Header.Set("X-Amz-Content-Sha256", hex.EncodeToString(metadata.contentSHA256Bytes))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
@ -356,6 +415,7 @@ func (c Client) newRequest(method string, metadata requestMetadata) (*http.Reque
|
|||
return req, nil
|
||||
}
|
||||
|
||||
// set User agent.
|
||||
func (c Client) setUserAgent(req *http.Request) {
|
||||
req.Header.Set("User-Agent", libraryUserAgent)
|
||||
if c.appInfo.appName != "" && c.appInfo.appVersion != "" {
|
||||
|
@ -363,12 +423,15 @@ func (c Client) setUserAgent(req *http.Request) {
|
|||
}
|
||||
}
|
||||
|
||||
// makeTargetURL make a new target url.
|
||||
func (c Client) makeTargetURL(bucketName, objectName string, queryValues url.Values) (*url.URL, error) {
|
||||
urlStr := c.endpointURL.Scheme + "://" + c.endpointURL.Host + "/"
|
||||
// Make URL only if bucketName is available, otherwise use the endpoint URL.
|
||||
// Make URL only if bucketName is available, otherwise use the
|
||||
// endpoint URL.
|
||||
if bucketName != "" {
|
||||
// If endpoint supports virtual host style use that always.
|
||||
// Currently only S3 and Google Cloud Storage would support this.
|
||||
// Currently only S3 and Google Cloud Storage would support
|
||||
// this.
|
||||
if isVirtualHostSupported(c.endpointURL) {
|
||||
urlStr = c.endpointURL.Scheme + "://" + bucketName + "." + c.endpointURL.Host + "/"
|
||||
if objectName != "" {
|
||||
|
@ -403,21 +466,17 @@ type CloudStorageClient interface {
|
|||
SetBucketACL(bucketName string, cannedACL BucketACL) error
|
||||
GetBucketACL(bucketName string) (BucketACL, error)
|
||||
|
||||
ListBuckets() ([]BucketStat, error)
|
||||
ListObjects(bucket, prefix string, recursive bool, doneCh <-chan struct{}) <-chan ObjectStat
|
||||
ListIncompleteUploads(bucket, prefix string, recursive bool, doneCh <-chan struct{}) <-chan ObjectMultipartStat
|
||||
ListBuckets() ([]BucketInfo, error)
|
||||
ListObjects(bucket, prefix string, recursive bool, doneCh <-chan struct{}) <-chan ObjectInfo
|
||||
ListIncompleteUploads(bucket, prefix string, recursive bool, doneCh <-chan struct{}) <-chan ObjectMultipartInfo
|
||||
|
||||
// Object Read/Write/Stat operations.
|
||||
GetObject(bucketName, objectName string) (reader io.ReadCloser, stat ObjectStat, err error)
|
||||
PutObject(bucketName, objectName string, data io.Reader, size int64, contentType string) (n int64, err error)
|
||||
StatObject(bucketName, objectName string) (ObjectStat, error)
|
||||
GetObject(bucketName, objectName string) (reader *Object, err error)
|
||||
PutObject(bucketName, objectName string, reader io.Reader, contentType string) (n int64, err error)
|
||||
StatObject(bucketName, objectName string) (ObjectInfo, error)
|
||||
RemoveObject(bucketName, objectName string) error
|
||||
RemoveIncompleteUpload(bucketName, objectName string) error
|
||||
|
||||
// Object Read/Write for sparse upload.
|
||||
GetObjectPartial(bucketName, objectName string) (reader ReadAtCloser, stat ObjectStat, err error)
|
||||
PutObjectPartial(bucketName, objectName string, data ReadAtCloser, size int64, contentType string) (n int64, err error)
|
||||
|
||||
// File to Object API.
|
||||
FPutObject(bucketName, objectName, filePath, contentType string) (n int64, err error)
|
||||
FGetObject(bucketName, objectName, filePath string) error
|
||||
|
|
751
Godeps/_workspace/src/github.com/minio/minio-go/api_functional_v2_test.go
generated
vendored
Normal file
751
Godeps/_workspace/src/github.com/minio/minio-go/api_functional_v2_test.go
generated
vendored
Normal file
|
@ -0,0 +1,751 @@
|
|||
/*
|
||||
* Minio Go Library for Amazon S3 Compatible Cloud Storage (C) 2015 Minio, Inc.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package minio_test
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
crand "crypto/rand"
|
||||
"errors"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"math/rand"
|
||||
"net/http"
|
||||
"os"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/minio/minio-go"
|
||||
)
|
||||
|
||||
// Tests removing partially uploaded objects.
|
||||
func TestRemovePartiallyUploadedV2(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("skipping function tests for short runs")
|
||||
}
|
||||
|
||||
// Seed random based on current time.
|
||||
rand.Seed(time.Now().Unix())
|
||||
|
||||
// Connect and make sure bucket exists.
|
||||
c, err := minio.NewV2(
|
||||
"s3.amazonaws.com",
|
||||
os.Getenv("ACCESS_KEY"),
|
||||
os.Getenv("SECRET_KEY"),
|
||||
false,
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatal("Error:", err)
|
||||
}
|
||||
|
||||
// Set user agent.
|
||||
c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0")
|
||||
|
||||
// Enable tracing, write to stdout.
|
||||
// c.TraceOn(os.Stderr)
|
||||
|
||||
// Generate a new random bucket name.
|
||||
bucketName := randString(60, rand.NewSource(time.Now().UnixNano()))
|
||||
|
||||
// make a new bucket.
|
||||
err = c.MakeBucket(bucketName, "private", "us-east-1")
|
||||
if err != nil {
|
||||
t.Fatal("Error:", err, bucketName)
|
||||
}
|
||||
|
||||
reader, writer := io.Pipe()
|
||||
go func() {
|
||||
i := 0
|
||||
for i < 25 {
|
||||
_, err = io.CopyN(writer, crand.Reader, 128*1024)
|
||||
if err != nil {
|
||||
t.Fatal("Error:", err, bucketName)
|
||||
}
|
||||
i++
|
||||
}
|
||||
writer.CloseWithError(errors.New("Proactively closed to be verified later."))
|
||||
}()
|
||||
|
||||
objectName := bucketName + "-resumable"
|
||||
_, err = c.PutObject(bucketName, objectName, reader, "application/octet-stream")
|
||||
if err == nil {
|
||||
t.Fatal("Error: PutObject should fail.")
|
||||
}
|
||||
if err.Error() != "Proactively closed to be verified later." {
|
||||
t.Fatal("Error:", err)
|
||||
}
|
||||
err = c.RemoveIncompleteUpload(bucketName, objectName)
|
||||
if err != nil {
|
||||
t.Fatal("Error:", err)
|
||||
}
|
||||
err = c.RemoveBucket(bucketName)
|
||||
if err != nil {
|
||||
t.Fatal("Error:", err)
|
||||
}
|
||||
}
|
||||
|
||||
// Tests resumable file based put object multipart upload.
|
||||
func TestResumableFPutObjectV2(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("skipping functional tests for the short runs")
|
||||
}
|
||||
|
||||
// Seed random based on current time.
|
||||
rand.Seed(time.Now().Unix())
|
||||
|
||||
// Connect and make sure bucket exists.
|
||||
c, err := minio.New(
|
||||
"s3.amazonaws.com",
|
||||
os.Getenv("ACCESS_KEY"),
|
||||
os.Getenv("SECRET_KEY"),
|
||||
false,
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatal("Error:", err)
|
||||
}
|
||||
|
||||
// Set user agent.
|
||||
c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0")
|
||||
|
||||
// Enable tracing, write to stdout.
|
||||
// c.TraceOn(os.Stderr)
|
||||
|
||||
// Generate a new random bucket name.
|
||||
bucketName := randString(60, rand.NewSource(time.Now().UnixNano()))
|
||||
|
||||
// make a new bucket.
|
||||
err = c.MakeBucket(bucketName, "private", "us-east-1")
|
||||
if err != nil {
|
||||
t.Fatal("Error:", err, bucketName)
|
||||
}
|
||||
|
||||
file, err := ioutil.TempFile(os.TempDir(), "resumable")
|
||||
if err != nil {
|
||||
t.Fatal("Error:", err)
|
||||
}
|
||||
|
||||
n, err := io.CopyN(file, crand.Reader, 11*1024*1024)
|
||||
if err != nil {
|
||||
t.Fatal("Error:", err)
|
||||
}
|
||||
if n != int64(11*1024*1024) {
|
||||
t.Fatalf("Error: number of bytes does not match, want %v, got %v\n", 11*1024*1024, n)
|
||||
}
|
||||
|
||||
objectName := bucketName + "-resumable"
|
||||
|
||||
n, err = c.FPutObject(bucketName, objectName, file.Name(), "application/octet-stream")
|
||||
if err != nil {
|
||||
t.Fatal("Error:", err)
|
||||
}
|
||||
if n != int64(11*1024*1024) {
|
||||
t.Fatalf("Error: number of bytes does not match, want %v, got %v\n", 11*1024*1024, n)
|
||||
}
|
||||
|
||||
// Close the file pro-actively for windows.
|
||||
file.Close()
|
||||
|
||||
err = c.RemoveObject(bucketName, objectName)
|
||||
if err != nil {
|
||||
t.Fatal("Error: ", err)
|
||||
}
|
||||
|
||||
err = c.RemoveBucket(bucketName)
|
||||
if err != nil {
|
||||
t.Fatal("Error:", err)
|
||||
}
|
||||
|
||||
err = os.Remove(file.Name())
|
||||
if err != nil {
|
||||
t.Fatal("Error:", err)
|
||||
}
|
||||
}
|
||||
|
||||
// Tests resumable put object multipart upload.
|
||||
func TestResumablePutObjectV2(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("skipping functional tests for the short runs")
|
||||
}
|
||||
|
||||
// Seed random based on current time.
|
||||
rand.Seed(time.Now().Unix())
|
||||
|
||||
// Connect and make sure bucket exists.
|
||||
c, err := minio.New(
|
||||
"s3.amazonaws.com",
|
||||
os.Getenv("ACCESS_KEY"),
|
||||
os.Getenv("SECRET_KEY"),
|
||||
false,
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatal("Error:", err)
|
||||
}
|
||||
|
||||
// Enable tracing, write to stderr.
|
||||
// c.TraceOn(os.Stderr)
|
||||
|
||||
// Set user agent.
|
||||
c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0")
|
||||
|
||||
// Generate a new random bucket name.
|
||||
bucketName := randString(60, rand.NewSource(time.Now().UnixNano()))
|
||||
|
||||
// make a new bucket.
|
||||
err = c.MakeBucket(bucketName, "private", "us-east-1")
|
||||
if err != nil {
|
||||
t.Fatal("Error:", err, bucketName)
|
||||
}
|
||||
|
||||
// generate 11MB
|
||||
buf := make([]byte, 11*1024*1024)
|
||||
|
||||
_, err = io.ReadFull(crand.Reader, buf)
|
||||
if err != nil {
|
||||
t.Fatal("Error:", err)
|
||||
}
|
||||
|
||||
objectName := bucketName + "-resumable"
|
||||
reader := bytes.NewReader(buf)
|
||||
n, err := c.PutObject(bucketName, objectName, reader, "application/octet-stream")
|
||||
if err != nil {
|
||||
t.Fatal("Error:", err, bucketName, objectName)
|
||||
}
|
||||
if n != int64(len(buf)) {
|
||||
t.Fatalf("Error: number of bytes does not match, want %v, got %v\n", len(buf), n)
|
||||
}
|
||||
|
||||
err = c.RemoveObject(bucketName, objectName)
|
||||
if err != nil {
|
||||
t.Fatal("Error: ", err)
|
||||
}
|
||||
|
||||
err = c.RemoveBucket(bucketName)
|
||||
if err != nil {
|
||||
t.Fatal("Error:", err)
|
||||
}
|
||||
}
|
||||
|
||||
// Tests get object ReaderSeeker interface methods.
|
||||
func TestGetObjectReadSeekFunctionalV2(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("skipping functional tests for short runs")
|
||||
}
|
||||
|
||||
// Seed random based on current time.
|
||||
rand.Seed(time.Now().Unix())
|
||||
|
||||
// Connect and make sure bucket exists.
|
||||
c, err := minio.New(
|
||||
"s3.amazonaws.com",
|
||||
os.Getenv("ACCESS_KEY"),
|
||||
os.Getenv("SECRET_KEY"),
|
||||
false,
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatal("Error:", err)
|
||||
}
|
||||
|
||||
// Enable tracing, write to stderr.
|
||||
// c.TraceOn(os.Stderr)
|
||||
|
||||
// Set user agent.
|
||||
c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0")
|
||||
|
||||
// Generate a new random bucket name.
|
||||
bucketName := randString(60, rand.NewSource(time.Now().UnixNano()))
|
||||
|
||||
// Make a new bucket.
|
||||
err = c.MakeBucket(bucketName, "private", "us-east-1")
|
||||
if err != nil {
|
||||
t.Fatal("Error:", err, bucketName)
|
||||
}
|
||||
|
||||
// Generate data more than 32K
|
||||
buf := make([]byte, rand.Intn(1<<20)+32*1024)
|
||||
|
||||
_, err = io.ReadFull(crand.Reader, buf)
|
||||
if err != nil {
|
||||
t.Fatal("Error:", err)
|
||||
}
|
||||
|
||||
// Save the data
|
||||
objectName := randString(60, rand.NewSource(time.Now().UnixNano()))
|
||||
n, err := c.PutObject(bucketName, objectName, bytes.NewReader(buf), "binary/octet-stream")
|
||||
if err != nil {
|
||||
t.Fatal("Error:", err, bucketName, objectName)
|
||||
}
|
||||
|
||||
if n != int64(len(buf)) {
|
||||
t.Fatalf("Error: number of bytes does not match, want %v, got %v\n", len(buf), n)
|
||||
}
|
||||
|
||||
// Read the data back
|
||||
r, err := c.GetObject(bucketName, objectName)
|
||||
if err != nil {
|
||||
t.Fatal("Error:", err, bucketName, objectName)
|
||||
}
|
||||
|
||||
st, err := r.Stat()
|
||||
if err != nil {
|
||||
t.Fatal("Error:", err, bucketName, objectName)
|
||||
}
|
||||
if st.Size != int64(len(buf)) {
|
||||
t.Fatalf("Error: number of bytes in stat does not match, want %v, got %v\n",
|
||||
len(buf), st.Size)
|
||||
}
|
||||
|
||||
offset := int64(2048)
|
||||
n, err = r.Seek(offset, 0)
|
||||
if err != nil {
|
||||
t.Fatal("Error:", err, offset)
|
||||
}
|
||||
if n != offset {
|
||||
t.Fatalf("Error: number of bytes seeked does not match, want %v, got %v\n",
|
||||
offset, n)
|
||||
}
|
||||
n, err = r.Seek(0, 1)
|
||||
if err != nil {
|
||||
t.Fatal("Error:", err)
|
||||
}
|
||||
if n != offset {
|
||||
t.Fatalf("Error: number of current seek does not match, want %v, got %v\n",
|
||||
offset, n)
|
||||
}
|
||||
_, err = r.Seek(offset, 2)
|
||||
if err == nil {
|
||||
t.Fatal("Error: seek on positive offset for whence '2' should error out")
|
||||
}
|
||||
n, err = r.Seek(-offset, 2)
|
||||
if err != nil {
|
||||
t.Fatal("Error:", err)
|
||||
}
|
||||
if n != 0 {
|
||||
t.Fatalf("Error: number of bytes seeked back does not match, want 0, got %v\n", n)
|
||||
}
|
||||
var buffer bytes.Buffer
|
||||
if _, err = io.CopyN(&buffer, r, st.Size); err != nil {
|
||||
t.Fatal("Error:", err)
|
||||
}
|
||||
if !bytes.Equal(buf, buffer.Bytes()) {
|
||||
t.Fatal("Error: Incorrect read bytes v/s original buffer.")
|
||||
}
|
||||
err = c.RemoveObject(bucketName, objectName)
|
||||
if err != nil {
|
||||
t.Fatal("Error: ", err)
|
||||
}
|
||||
err = c.RemoveBucket(bucketName)
|
||||
if err != nil {
|
||||
t.Fatal("Error:", err)
|
||||
}
|
||||
}
|
||||
|
||||
// Tests get object ReaderAt interface methods.
|
||||
func TestGetObjectReadAtFunctionalV2(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("skipping functional tests for the short runs")
|
||||
}
|
||||
|
||||
// Seed random based on current time.
|
||||
rand.Seed(time.Now().Unix())
|
||||
|
||||
// Connect and make sure bucket exists.
|
||||
c, err := minio.New(
|
||||
"s3.amazonaws.com",
|
||||
os.Getenv("ACCESS_KEY"),
|
||||
os.Getenv("SECRET_KEY"),
|
||||
false,
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatal("Error:", err)
|
||||
}
|
||||
|
||||
// Enable tracing, write to stderr.
|
||||
// c.TraceOn(os.Stderr)
|
||||
|
||||
// Set user agent.
|
||||
c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0")
|
||||
|
||||
// Generate a new random bucket name.
|
||||
bucketName := randString(60, rand.NewSource(time.Now().UnixNano()))
|
||||
|
||||
// Make a new bucket.
|
||||
err = c.MakeBucket(bucketName, "private", "us-east-1")
|
||||
if err != nil {
|
||||
t.Fatal("Error:", err, bucketName)
|
||||
}
|
||||
|
||||
// Generate data more than 32K
|
||||
buf := make([]byte, rand.Intn(1<<20)+32*1024)
|
||||
|
||||
_, err = io.ReadFull(crand.Reader, buf)
|
||||
if err != nil {
|
||||
t.Fatal("Error:", err)
|
||||
}
|
||||
|
||||
// Save the data
|
||||
objectName := randString(60, rand.NewSource(time.Now().UnixNano()))
|
||||
n, err := c.PutObject(bucketName, objectName, bytes.NewReader(buf), "binary/octet-stream")
|
||||
if err != nil {
|
||||
t.Fatal("Error:", err, bucketName, objectName)
|
||||
}
|
||||
|
||||
if n != int64(len(buf)) {
|
||||
t.Fatalf("Error: number of bytes does not match, want %v, got %v\n", len(buf), n)
|
||||
}
|
||||
|
||||
// Read the data back
|
||||
r, err := c.GetObject(bucketName, objectName)
|
||||
if err != nil {
|
||||
t.Fatal("Error:", err, bucketName, objectName)
|
||||
}
|
||||
|
||||
st, err := r.Stat()
|
||||
if err != nil {
|
||||
t.Fatal("Error:", err, bucketName, objectName)
|
||||
}
|
||||
if st.Size != int64(len(buf)) {
|
||||
t.Fatalf("Error: number of bytes in stat does not match, want %v, got %v\n",
|
||||
len(buf), st.Size)
|
||||
}
|
||||
|
||||
offset := int64(2048)
|
||||
|
||||
// Read directly
|
||||
buf2 := make([]byte, 512)
|
||||
buf3 := make([]byte, 512)
|
||||
buf4 := make([]byte, 512)
|
||||
|
||||
m, err := r.ReadAt(buf2, offset)
|
||||
if err != nil {
|
||||
t.Fatal("Error:", err, st.Size, len(buf2), offset)
|
||||
}
|
||||
if m != len(buf2) {
|
||||
t.Fatalf("Error: ReadAt read shorter bytes before reaching EOF, want %v, got %v\n", m, len(buf2))
|
||||
}
|
||||
if !bytes.Equal(buf2, buf[offset:offset+512]) {
|
||||
t.Fatal("Error: Incorrect read between two ReadAt from same offset.")
|
||||
}
|
||||
offset += 512
|
||||
m, err = r.ReadAt(buf3, offset)
|
||||
if err != nil {
|
||||
t.Fatal("Error:", err, st.Size, len(buf3), offset)
|
||||
}
|
||||
if m != len(buf3) {
|
||||
t.Fatalf("Error: ReadAt read shorter bytes before reaching EOF, want %v, got %v\n", m, len(buf3))
|
||||
}
|
||||
if !bytes.Equal(buf3, buf[offset:offset+512]) {
|
||||
t.Fatal("Error: Incorrect read between two ReadAt from same offset.")
|
||||
}
|
||||
offset += 512
|
||||
m, err = r.ReadAt(buf4, offset)
|
||||
if err != nil {
|
||||
t.Fatal("Error:", err, st.Size, len(buf4), offset)
|
||||
}
|
||||
if m != len(buf4) {
|
||||
t.Fatalf("Error: ReadAt read shorter bytes before reaching EOF, want %v, got %v\n", m, len(buf4))
|
||||
}
|
||||
if !bytes.Equal(buf4, buf[offset:offset+512]) {
|
||||
t.Fatal("Error: Incorrect read between two ReadAt from same offset.")
|
||||
}
|
||||
|
||||
buf5 := make([]byte, n)
|
||||
// Read the whole object.
|
||||
m, err = r.ReadAt(buf5, 0)
|
||||
if err != nil {
|
||||
if err != io.EOF {
|
||||
t.Fatal("Error:", err, len(buf5))
|
||||
}
|
||||
}
|
||||
if m != len(buf5) {
|
||||
t.Fatalf("Error: ReadAt read shorter bytes before reaching EOF, want %v, got %v\n", m, len(buf5))
|
||||
}
|
||||
if !bytes.Equal(buf, buf5) {
|
||||
t.Fatal("Error: Incorrect data read in GetObject, than what was previously upoaded.")
|
||||
}
|
||||
|
||||
buf6 := make([]byte, n+1)
|
||||
// Read the whole object and beyond.
|
||||
_, err = r.ReadAt(buf6, 0)
|
||||
if err != nil {
|
||||
if err != io.EOF {
|
||||
t.Fatal("Error:", err, len(buf6))
|
||||
}
|
||||
}
|
||||
err = c.RemoveObject(bucketName, objectName)
|
||||
if err != nil {
|
||||
t.Fatal("Error: ", err)
|
||||
}
|
||||
err = c.RemoveBucket(bucketName)
|
||||
if err != nil {
|
||||
t.Fatal("Error:", err)
|
||||
}
|
||||
}
|
||||
|
||||
// Tests comprehensive list of all methods.
|
||||
func TestFunctionalV2(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("skipping functional tests for the short runs")
|
||||
}
|
||||
|
||||
// Seed random based on current time.
|
||||
rand.Seed(time.Now().Unix())
|
||||
|
||||
c, err := minio.New(
|
||||
"s3.amazonaws.com",
|
||||
os.Getenv("ACCESS_KEY"),
|
||||
os.Getenv("SECRET_KEY"),
|
||||
false,
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatal("Error:", err)
|
||||
}
|
||||
|
||||
// Enable to debug
|
||||
// c.TraceOn(os.Stderr)
|
||||
|
||||
// Set user agent.
|
||||
c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0")
|
||||
|
||||
// Generate a new random bucket name.
|
||||
bucketName := randString(60, rand.NewSource(time.Now().UnixNano()))
|
||||
|
||||
// Make a new bucket.
|
||||
err = c.MakeBucket(bucketName, "private", "us-east-1")
|
||||
if err != nil {
|
||||
t.Fatal("Error:", err, bucketName)
|
||||
}
|
||||
|
||||
// Generate a random file name.
|
||||
fileName := randString(60, rand.NewSource(time.Now().UnixNano()))
|
||||
file, err := os.Create(fileName)
|
||||
if err != nil {
|
||||
t.Fatal("Error:", err)
|
||||
}
|
||||
var totalSize int64
|
||||
for i := 0; i < 3; i++ {
|
||||
buf := make([]byte, rand.Intn(1<<19))
|
||||
n, err := file.Write(buf)
|
||||
if err != nil {
|
||||
t.Fatal("Error:", err)
|
||||
}
|
||||
totalSize += int64(n)
|
||||
}
|
||||
file.Close()
|
||||
|
||||
// Verify if bucket exits and you have access.
|
||||
err = c.BucketExists(bucketName)
|
||||
if err != nil {
|
||||
t.Fatal("Error:", err, bucketName)
|
||||
}
|
||||
|
||||
// Make the bucket 'public read/write'.
|
||||
err = c.SetBucketACL(bucketName, "public-read-write")
|
||||
if err != nil {
|
||||
t.Fatal("Error:", err)
|
||||
}
|
||||
|
||||
// Get the previously set acl.
|
||||
acl, err := c.GetBucketACL(bucketName)
|
||||
if err != nil {
|
||||
t.Fatal("Error:", err)
|
||||
}
|
||||
|
||||
// ACL must be 'public read/write'.
|
||||
if acl != minio.BucketACL("public-read-write") {
|
||||
t.Fatal("Error:", acl)
|
||||
}
|
||||
|
||||
// List all buckets.
|
||||
buckets, err := c.ListBuckets()
|
||||
if len(buckets) == 0 {
|
||||
t.Fatal("Error: list buckets cannot be empty", buckets)
|
||||
}
|
||||
if err != nil {
|
||||
t.Fatal("Error:", err)
|
||||
}
|
||||
|
||||
// Verify if previously created bucket is listed in list buckets.
|
||||
bucketFound := false
|
||||
for _, bucket := range buckets {
|
||||
if bucket.Name == bucketName {
|
||||
bucketFound = true
|
||||
}
|
||||
}
|
||||
|
||||
// If bucket not found error out.
|
||||
if !bucketFound {
|
||||
t.Fatal("Error: bucket ", bucketName, "not found")
|
||||
}
|
||||
|
||||
objectName := bucketName + "unique"
|
||||
|
||||
// Generate data
|
||||
buf := make([]byte, rand.Intn(1<<19))
|
||||
_, err = io.ReadFull(crand.Reader, buf)
|
||||
if err != nil {
|
||||
t.Fatal("Error: ", err)
|
||||
}
|
||||
|
||||
n, err := c.PutObject(bucketName, objectName, bytes.NewReader(buf), "")
|
||||
if err != nil {
|
||||
t.Fatal("Error: ", err)
|
||||
}
|
||||
if n != int64(len(buf)) {
|
||||
t.Fatal("Error: bad length ", n, len(buf))
|
||||
}
|
||||
|
||||
n, err = c.PutObject(bucketName, objectName+"-nolength", bytes.NewReader(buf), "binary/octet-stream")
|
||||
if err != nil {
|
||||
t.Fatal("Error:", err, bucketName, objectName+"-nolength")
|
||||
}
|
||||
|
||||
if n != int64(len(buf)) {
|
||||
t.Fatalf("Error: number of bytes does not match, want %v, got %v\n", len(buf), n)
|
||||
}
|
||||
|
||||
// Instantiate a done channel to close all listing.
|
||||
doneCh := make(chan struct{})
|
||||
defer close(doneCh)
|
||||
|
||||
objFound := false
|
||||
isRecursive := true // Recursive is true.
|
||||
for obj := range c.ListObjects(bucketName, objectName, isRecursive, doneCh) {
|
||||
if obj.Key == objectName {
|
||||
objFound = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !objFound {
|
||||
t.Fatal("Error: object " + objectName + " not found.")
|
||||
}
|
||||
|
||||
incompObjNotFound := true
|
||||
for objIncompl := range c.ListIncompleteUploads(bucketName, objectName, isRecursive, doneCh) {
|
||||
if objIncompl.Key != "" {
|
||||
incompObjNotFound = false
|
||||
break
|
||||
}
|
||||
}
|
||||
if !incompObjNotFound {
|
||||
t.Fatal("Error: unexpected dangling incomplete upload found.")
|
||||
}
|
||||
|
||||
newReader, err := c.GetObject(bucketName, objectName)
|
||||
if err != nil {
|
||||
t.Fatal("Error: ", err)
|
||||
}
|
||||
|
||||
newReadBytes, err := ioutil.ReadAll(newReader)
|
||||
if err != nil {
|
||||
t.Fatal("Error: ", err)
|
||||
}
|
||||
|
||||
if !bytes.Equal(newReadBytes, buf) {
|
||||
t.Fatal("Error: bytes mismatch.")
|
||||
}
|
||||
|
||||
err = c.FGetObject(bucketName, objectName, fileName+"-f")
|
||||
if err != nil {
|
||||
t.Fatal("Error: ", err)
|
||||
}
|
||||
|
||||
presignedGetURL, err := c.PresignedGetObject(bucketName, objectName, 3600*time.Second)
|
||||
if err != nil {
|
||||
t.Fatal("Error: ", err)
|
||||
}
|
||||
|
||||
resp, err := http.Get(presignedGetURL)
|
||||
if err != nil {
|
||||
t.Fatal("Error: ", err)
|
||||
}
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
t.Fatal("Error: ", resp.Status)
|
||||
}
|
||||
newPresignedBytes, err := ioutil.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
t.Fatal("Error: ", err)
|
||||
}
|
||||
if !bytes.Equal(newPresignedBytes, buf) {
|
||||
t.Fatal("Error: bytes mismatch.")
|
||||
}
|
||||
|
||||
presignedPutURL, err := c.PresignedPutObject(bucketName, objectName+"-presigned", 3600*time.Second)
|
||||
if err != nil {
|
||||
t.Fatal("Error: ", err)
|
||||
}
|
||||
buf = make([]byte, rand.Intn(1<<20))
|
||||
_, err = io.ReadFull(crand.Reader, buf)
|
||||
if err != nil {
|
||||
t.Fatal("Error: ", err)
|
||||
}
|
||||
req, err := http.NewRequest("PUT", presignedPutURL, bytes.NewReader(buf))
|
||||
if err != nil {
|
||||
t.Fatal("Error: ", err)
|
||||
}
|
||||
httpClient := &http.Client{}
|
||||
resp, err = httpClient.Do(req)
|
||||
if err != nil {
|
||||
t.Fatal("Error: ", err)
|
||||
}
|
||||
|
||||
newReader, err = c.GetObject(bucketName, objectName+"-presigned")
|
||||
if err != nil {
|
||||
t.Fatal("Error: ", err)
|
||||
}
|
||||
|
||||
newReadBytes, err = ioutil.ReadAll(newReader)
|
||||
if err != nil {
|
||||
t.Fatal("Error: ", err)
|
||||
}
|
||||
|
||||
if !bytes.Equal(newReadBytes, buf) {
|
||||
t.Fatal("Error: bytes mismatch.")
|
||||
}
|
||||
|
||||
err = c.RemoveObject(bucketName, objectName)
|
||||
if err != nil {
|
||||
t.Fatal("Error: ", err)
|
||||
}
|
||||
err = c.RemoveObject(bucketName, objectName+"-f")
|
||||
if err != nil {
|
||||
t.Fatal("Error: ", err)
|
||||
}
|
||||
err = c.RemoveObject(bucketName, objectName+"-nolength")
|
||||
if err != nil {
|
||||
t.Fatal("Error: ", err)
|
||||
}
|
||||
err = c.RemoveObject(bucketName, objectName+"-presigned")
|
||||
if err != nil {
|
||||
t.Fatal("Error: ", err)
|
||||
}
|
||||
err = c.RemoveBucket(bucketName)
|
||||
if err != nil {
|
||||
t.Fatal("Error:", err)
|
||||
}
|
||||
err = c.RemoveBucket(bucketName)
|
||||
if err == nil {
|
||||
t.Fatal("Error:")
|
||||
}
|
||||
if err.Error() != "The specified bucket does not exist" {
|
||||
t.Fatal("Error: ", err)
|
||||
}
|
||||
if err = os.Remove(fileName); err != nil {
|
||||
t.Fatal("Error: ", err)
|
||||
}
|
||||
if err = os.Remove(fileName + "-f"); err != nil {
|
||||
t.Fatal("Error: ", err)
|
||||
}
|
||||
}
|
|
@ -19,6 +19,7 @@ package minio_test
|
|||
import (
|
||||
"bytes"
|
||||
crand "crypto/rand"
|
||||
"errors"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"math/rand"
|
||||
|
@ -54,9 +55,10 @@ func randString(n int, src rand.Source) string {
|
|||
return string(b[0:30])
|
||||
}
|
||||
|
||||
func TestResumableFPutObject(t *testing.T) {
|
||||
// Tests removing partially uploaded objects.
|
||||
func TestRemovePartiallyUploaded(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("skipping resumable tests with short runs")
|
||||
t.Skip("skipping function tests for short runs")
|
||||
}
|
||||
|
||||
// Seed random based on current time.
|
||||
|
@ -64,9 +66,9 @@ func TestResumableFPutObject(t *testing.T) {
|
|||
|
||||
// Connect and make sure bucket exists.
|
||||
c, err := minio.New(
|
||||
"play.minio.io:9002",
|
||||
"Q3AM3UQ867SPQQA43P2F",
|
||||
"zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG",
|
||||
"s3.amazonaws.com",
|
||||
os.Getenv("ACCESS_KEY"),
|
||||
os.Getenv("SECRET_KEY"),
|
||||
false,
|
||||
)
|
||||
if err != nil {
|
||||
|
@ -77,12 +79,78 @@ func TestResumableFPutObject(t *testing.T) {
|
|||
c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0")
|
||||
|
||||
// Enable tracing, write to stdout.
|
||||
// c.TraceOn(nil)
|
||||
// c.TraceOn(os.Stderr)
|
||||
|
||||
// Generate a new random bucket name.
|
||||
bucketName := randString(60, rand.NewSource(time.Now().UnixNano()))
|
||||
|
||||
// make a new bucket.
|
||||
// Make a new bucket.
|
||||
err = c.MakeBucket(bucketName, "private", "us-east-1")
|
||||
if err != nil {
|
||||
t.Fatal("Error:", err, bucketName)
|
||||
}
|
||||
|
||||
reader, writer := io.Pipe()
|
||||
go func() {
|
||||
i := 0
|
||||
for i < 25 {
|
||||
_, err = io.CopyN(writer, crand.Reader, 128*1024)
|
||||
if err != nil {
|
||||
t.Fatal("Error:", err, bucketName)
|
||||
}
|
||||
i++
|
||||
}
|
||||
writer.CloseWithError(errors.New("Proactively closed to be verified later."))
|
||||
}()
|
||||
|
||||
objectName := bucketName + "-resumable"
|
||||
_, err = c.PutObject(bucketName, objectName, reader, "application/octet-stream")
|
||||
if err == nil {
|
||||
t.Fatal("Error: PutObject should fail.")
|
||||
}
|
||||
if err.Error() != "Proactively closed to be verified later." {
|
||||
t.Fatal("Error:", err)
|
||||
}
|
||||
err = c.RemoveIncompleteUpload(bucketName, objectName)
|
||||
if err != nil {
|
||||
t.Fatal("Error:", err)
|
||||
}
|
||||
err = c.RemoveBucket(bucketName)
|
||||
if err != nil {
|
||||
t.Fatal("Error:", err)
|
||||
}
|
||||
}
|
||||
|
||||
// Tests resumable file based put object multipart upload.
|
||||
func TestResumableFPutObject(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("skipping functional tests for the short runs")
|
||||
}
|
||||
|
||||
// Seed random based on current time.
|
||||
rand.Seed(time.Now().Unix())
|
||||
|
||||
// Connect and make sure bucket exists.
|
||||
c, err := minio.New(
|
||||
"s3.amazonaws.com",
|
||||
os.Getenv("ACCESS_KEY"),
|
||||
os.Getenv("SECRET_KEY"),
|
||||
false,
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatal("Error:", err)
|
||||
}
|
||||
|
||||
// Set user agent.
|
||||
c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0")
|
||||
|
||||
// Enable tracing, write to stdout.
|
||||
// c.TraceOn(os.Stderr)
|
||||
|
||||
// Generate a new random bucket name.
|
||||
bucketName := randString(60, rand.NewSource(time.Now().UnixNano()))
|
||||
|
||||
// Make a new bucket.
|
||||
err = c.MakeBucket(bucketName, "private", "us-east-1")
|
||||
if err != nil {
|
||||
t.Fatal("Error:", err, bucketName)
|
||||
|
@ -93,7 +161,10 @@ func TestResumableFPutObject(t *testing.T) {
|
|||
t.Fatal("Error:", err)
|
||||
}
|
||||
|
||||
n, _ := io.CopyN(file, crand.Reader, 11*1024*1024)
|
||||
n, err := io.CopyN(file, crand.Reader, 11*1024*1024)
|
||||
if err != nil {
|
||||
t.Fatal("Error:", err)
|
||||
}
|
||||
if n != int64(11*1024*1024) {
|
||||
t.Fatalf("Error: number of bytes does not match, want %v, got %v\n", 11*1024*1024, n)
|
||||
}
|
||||
|
@ -127,9 +198,10 @@ func TestResumableFPutObject(t *testing.T) {
|
|||
}
|
||||
}
|
||||
|
||||
// Tests resumable put object multipart upload.
|
||||
func TestResumablePutObject(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("skipping resumable tests with short runs")
|
||||
t.Skip("skipping functional tests for the short runs")
|
||||
}
|
||||
|
||||
// Seed random based on current time.
|
||||
|
@ -137,31 +209,31 @@ func TestResumablePutObject(t *testing.T) {
|
|||
|
||||
// Connect and make sure bucket exists.
|
||||
c, err := minio.New(
|
||||
"play.minio.io:9002",
|
||||
"Q3AM3UQ867SPQQA43P2F",
|
||||
"zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG",
|
||||
"s3.amazonaws.com",
|
||||
os.Getenv("ACCESS_KEY"),
|
||||
os.Getenv("SECRET_KEY"),
|
||||
false,
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatal("Error:", err)
|
||||
}
|
||||
|
||||
// Enable tracing, write to stderr.
|
||||
// c.TraceOn(os.Stderr)
|
||||
|
||||
// Set user agent.
|
||||
c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0")
|
||||
|
||||
// Enable tracing, write to stdout.
|
||||
// c.TraceOn(nil)
|
||||
|
||||
// Generate a new random bucket name.
|
||||
bucketName := randString(60, rand.NewSource(time.Now().UnixNano()))
|
||||
|
||||
// make a new bucket.
|
||||
// Make a new bucket.
|
||||
err = c.MakeBucket(bucketName, "private", "us-east-1")
|
||||
if err != nil {
|
||||
t.Fatal("Error:", err, bucketName)
|
||||
}
|
||||
|
||||
// generate 11MB
|
||||
// Generate 11MB
|
||||
buf := make([]byte, 11*1024*1024)
|
||||
|
||||
_, err = io.ReadFull(crand.Reader, buf)
|
||||
|
@ -171,7 +243,7 @@ func TestResumablePutObject(t *testing.T) {
|
|||
|
||||
objectName := bucketName + "-resumable"
|
||||
reader := bytes.NewReader(buf)
|
||||
n, err := c.PutObject(bucketName, objectName, reader, int64(reader.Len()), "application/octet-stream")
|
||||
n, err := c.PutObject(bucketName, objectName, reader, "application/octet-stream")
|
||||
if err != nil {
|
||||
t.Fatal("Error:", err, bucketName, objectName)
|
||||
}
|
||||
|
@ -190,37 +262,42 @@ func TestResumablePutObject(t *testing.T) {
|
|||
}
|
||||
}
|
||||
|
||||
func TestGetObjectPartialFunctional(t *testing.T) {
|
||||
// Tests get object ReaderSeeker interface methods.
|
||||
func TestGetObjectReadSeekFunctional(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("skipping functional tests for short runs")
|
||||
}
|
||||
|
||||
// Seed random based on current time.
|
||||
rand.Seed(time.Now().Unix())
|
||||
|
||||
// Connect and make sure bucket exists.
|
||||
c, err := minio.New(
|
||||
"play.minio.io:9002",
|
||||
"Q3AM3UQ867SPQQA43P2F",
|
||||
"zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG",
|
||||
"s3.amazonaws.com",
|
||||
os.Getenv("ACCESS_KEY"),
|
||||
os.Getenv("SECRET_KEY"),
|
||||
false,
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatal("Error:", err)
|
||||
}
|
||||
|
||||
// Enable tracing, write to stderr.
|
||||
// c.TraceOn(os.Stderr)
|
||||
|
||||
// Set user agent.
|
||||
c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0")
|
||||
|
||||
// Enable tracing, write to stdout.
|
||||
// c.TraceOn(nil)
|
||||
|
||||
// Generate a new random bucket name.
|
||||
bucketName := randString(60, rand.NewSource(time.Now().UnixNano()))
|
||||
|
||||
// make a new bucket.
|
||||
// Make a new bucket.
|
||||
err = c.MakeBucket(bucketName, "private", "us-east-1")
|
||||
if err != nil {
|
||||
t.Fatal("Error:", err, bucketName)
|
||||
}
|
||||
|
||||
// generate data more than 32K
|
||||
// Generate data more than 32K
|
||||
buf := make([]byte, rand.Intn(1<<20)+32*1024)
|
||||
|
||||
_, err = io.ReadFull(crand.Reader, buf)
|
||||
|
@ -228,9 +305,123 @@ func TestGetObjectPartialFunctional(t *testing.T) {
|
|||
t.Fatal("Error:", err)
|
||||
}
|
||||
|
||||
// save the data
|
||||
// Save the data
|
||||
objectName := randString(60, rand.NewSource(time.Now().UnixNano()))
|
||||
n, err := c.PutObject(bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), "binary/octet-stream")
|
||||
n, err := c.PutObject(bucketName, objectName, bytes.NewReader(buf), "binary/octet-stream")
|
||||
if err != nil {
|
||||
t.Fatal("Error:", err, bucketName, objectName)
|
||||
}
|
||||
|
||||
if n != int64(len(buf)) {
|
||||
t.Fatalf("Error: number of bytes does not match, want %v, got %v\n", len(buf), n)
|
||||
}
|
||||
|
||||
// Read the data back
|
||||
r, err := c.GetObject(bucketName, objectName)
|
||||
if err != nil {
|
||||
t.Fatal("Error:", err, bucketName, objectName)
|
||||
}
|
||||
|
||||
st, err := r.Stat()
|
||||
if err != nil {
|
||||
t.Fatal("Error:", err, bucketName, objectName)
|
||||
}
|
||||
if st.Size != int64(len(buf)) {
|
||||
t.Fatalf("Error: number of bytes in stat does not match, want %v, got %v\n",
|
||||
len(buf), st.Size)
|
||||
}
|
||||
|
||||
offset := int64(2048)
|
||||
n, err = r.Seek(offset, 0)
|
||||
if err != nil {
|
||||
t.Fatal("Error:", err, offset)
|
||||
}
|
||||
if n != offset {
|
||||
t.Fatalf("Error: number of bytes seeked does not match, want %v, got %v\n",
|
||||
offset, n)
|
||||
}
|
||||
n, err = r.Seek(0, 1)
|
||||
if err != nil {
|
||||
t.Fatal("Error:", err)
|
||||
}
|
||||
if n != offset {
|
||||
t.Fatalf("Error: number of current seek does not match, want %v, got %v\n",
|
||||
offset, n)
|
||||
}
|
||||
_, err = r.Seek(offset, 2)
|
||||
if err == nil {
|
||||
t.Fatal("Error: seek on positive offset for whence '2' should error out")
|
||||
}
|
||||
n, err = r.Seek(-offset, 2)
|
||||
if err != nil {
|
||||
t.Fatal("Error:", err)
|
||||
}
|
||||
if n != 0 {
|
||||
t.Fatalf("Error: number of bytes seeked back does not match, want 0, got %v\n", n)
|
||||
}
|
||||
var buffer bytes.Buffer
|
||||
if _, err = io.CopyN(&buffer, r, st.Size); err != nil {
|
||||
t.Fatal("Error:", err)
|
||||
}
|
||||
if !bytes.Equal(buf, buffer.Bytes()) {
|
||||
t.Fatal("Error: Incorrect read bytes v/s original buffer.")
|
||||
}
|
||||
err = c.RemoveObject(bucketName, objectName)
|
||||
if err != nil {
|
||||
t.Fatal("Error: ", err)
|
||||
}
|
||||
err = c.RemoveBucket(bucketName)
|
||||
if err != nil {
|
||||
t.Fatal("Error:", err)
|
||||
}
|
||||
}
|
||||
|
||||
// Tests get object ReaderAt interface methods.
|
||||
func TestGetObjectReadAtFunctional(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("skipping functional tests for the short runs")
|
||||
}
|
||||
|
||||
// Seed random based on current time.
|
||||
rand.Seed(time.Now().Unix())
|
||||
|
||||
// Connect and make sure bucket exists.
|
||||
c, err := minio.New(
|
||||
"s3.amazonaws.com",
|
||||
os.Getenv("ACCESS_KEY"),
|
||||
os.Getenv("SECRET_KEY"),
|
||||
false,
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatal("Error:", err)
|
||||
}
|
||||
|
||||
// Enable tracing, write to stderr.
|
||||
// c.TraceOn(os.Stderr)
|
||||
|
||||
// Set user agent.
|
||||
c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0")
|
||||
|
||||
// Generate a new random bucket name.
|
||||
bucketName := randString(60, rand.NewSource(time.Now().UnixNano()))
|
||||
|
||||
// Make a new bucket.
|
||||
err = c.MakeBucket(bucketName, "private", "us-east-1")
|
||||
if err != nil {
|
||||
t.Fatal("Error:", err, bucketName)
|
||||
}
|
||||
|
||||
// Generate data more than 32K
|
||||
buf := make([]byte, rand.Intn(1<<20)+32*1024)
|
||||
|
||||
_, err = io.ReadFull(crand.Reader, buf)
|
||||
if err != nil {
|
||||
t.Fatal("Error:", err)
|
||||
}
|
||||
|
||||
// Save the data
|
||||
objectName := randString(60, rand.NewSource(time.Now().UnixNano()))
|
||||
n, err := c.PutObject(bucketName, objectName, bytes.NewReader(buf), "binary/octet-stream")
|
||||
if err != nil {
|
||||
t.Fatal("Error:", err, bucketName, objectName)
|
||||
}
|
||||
|
@ -240,11 +431,15 @@ func TestGetObjectPartialFunctional(t *testing.T) {
|
|||
}
|
||||
|
||||
// read the data back
|
||||
r, st, err := c.GetObjectPartial(bucketName, objectName)
|
||||
r, err := c.GetObject(bucketName, objectName)
|
||||
if err != nil {
|
||||
t.Fatal("Error:", err, bucketName, objectName)
|
||||
}
|
||||
|
||||
st, err := r.Stat()
|
||||
if err != nil {
|
||||
t.Fatal("Error:", err, bucketName, objectName)
|
||||
}
|
||||
if st.Size != int64(len(buf)) {
|
||||
t.Fatalf("Error: number of bytes in stat does not match, want %v, got %v\n",
|
||||
len(buf), st.Size)
|
||||
|
@ -323,36 +518,41 @@ func TestGetObjectPartialFunctional(t *testing.T) {
|
|||
}
|
||||
}
|
||||
|
||||
// Tests comprehensive list of all methods.
|
||||
func TestFunctional(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("skipping functional tests for the short runs")
|
||||
}
|
||||
|
||||
// Seed random based on current time.
|
||||
rand.Seed(time.Now().Unix())
|
||||
|
||||
c, err := minio.New(
|
||||
"play.minio.io:9002",
|
||||
"Q3AM3UQ867SPQQA43P2F",
|
||||
"zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG",
|
||||
"s3.amazonaws.com",
|
||||
os.Getenv("ACCESS_KEY"),
|
||||
os.Getenv("SECRET_KEY"),
|
||||
false,
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatal("Error:", err)
|
||||
}
|
||||
|
||||
// Enable to debug
|
||||
// c.TraceOn(os.Stderr)
|
||||
|
||||
// Set user agent.
|
||||
c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0")
|
||||
|
||||
// Enable tracing, write to stdout.
|
||||
// c.TraceOn(nil)
|
||||
|
||||
// Generate a new random bucket name.
|
||||
bucketName := randString(60, rand.NewSource(time.Now().UnixNano()))
|
||||
|
||||
// make a new bucket.
|
||||
// Make a new bucket.
|
||||
err = c.MakeBucket(bucketName, "private", "us-east-1")
|
||||
if err != nil {
|
||||
t.Fatal("Error:", err, bucketName)
|
||||
}
|
||||
|
||||
// generate a random file name.
|
||||
// Generate a random file name.
|
||||
fileName := randString(60, rand.NewSource(time.Now().UnixNano()))
|
||||
file, err := os.Create(fileName)
|
||||
if err != nil {
|
||||
|
@ -369,31 +569,34 @@ func TestFunctional(t *testing.T) {
|
|||
}
|
||||
file.Close()
|
||||
|
||||
// verify if bucket exits and you have access.
|
||||
// Verify if bucket exits and you have access.
|
||||
err = c.BucketExists(bucketName)
|
||||
if err != nil {
|
||||
t.Fatal("Error:", err, bucketName)
|
||||
}
|
||||
|
||||
// make the bucket 'public read/write'.
|
||||
// Make the bucket 'public read/write'.
|
||||
err = c.SetBucketACL(bucketName, "public-read-write")
|
||||
if err != nil {
|
||||
t.Fatal("Error:", err)
|
||||
}
|
||||
|
||||
// get the previously set acl.
|
||||
// Get the previously set acl.
|
||||
acl, err := c.GetBucketACL(bucketName)
|
||||
if err != nil {
|
||||
t.Fatal("Error:", err)
|
||||
}
|
||||
|
||||
// acl must be 'public read/write'.
|
||||
// ACL must be 'public read/write'.
|
||||
if acl != minio.BucketACL("public-read-write") {
|
||||
t.Fatal("Error:", acl)
|
||||
}
|
||||
|
||||
// list all buckets.
|
||||
// List all buckets.
|
||||
buckets, err := c.ListBuckets()
|
||||
if len(buckets) == 0 {
|
||||
t.Fatal("Error: list buckets cannot be empty", buckets)
|
||||
}
|
||||
if err != nil {
|
||||
t.Fatal("Error:", err)
|
||||
}
|
||||
|
@ -413,14 +616,14 @@ func TestFunctional(t *testing.T) {
|
|||
|
||||
objectName := bucketName + "unique"
|
||||
|
||||
// generate data
|
||||
// Generate data
|
||||
buf := make([]byte, rand.Intn(1<<19))
|
||||
_, err = io.ReadFull(crand.Reader, buf)
|
||||
if err != nil {
|
||||
t.Fatal("Error: ", err)
|
||||
}
|
||||
|
||||
n, err := c.PutObject(bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), "")
|
||||
n, err := c.PutObject(bucketName, objectName, bytes.NewReader(buf), "")
|
||||
if err != nil {
|
||||
t.Fatal("Error: ", err)
|
||||
}
|
||||
|
@ -428,7 +631,7 @@ func TestFunctional(t *testing.T) {
|
|||
t.Fatal("Error: bad length ", n, len(buf))
|
||||
}
|
||||
|
||||
n, err = c.PutObject(bucketName, objectName+"-nolength", bytes.NewReader(buf), -1, "binary/octet-stream")
|
||||
n, err = c.PutObject(bucketName, objectName+"-nolength", bytes.NewReader(buf), "binary/octet-stream")
|
||||
if err != nil {
|
||||
t.Fatal("Error:", err, bucketName, objectName+"-nolength")
|
||||
}
|
||||
|
@ -437,7 +640,34 @@ func TestFunctional(t *testing.T) {
|
|||
t.Fatalf("Error: number of bytes does not match, want %v, got %v\n", len(buf), n)
|
||||
}
|
||||
|
||||
newReader, _, err := c.GetObject(bucketName, objectName)
|
||||
// Instantiate a done channel to close all listing.
|
||||
doneCh := make(chan struct{})
|
||||
defer close(doneCh)
|
||||
|
||||
objFound := false
|
||||
isRecursive := true // Recursive is true.
|
||||
for obj := range c.ListObjects(bucketName, objectName, isRecursive, doneCh) {
|
||||
if obj.Key == objectName {
|
||||
objFound = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !objFound {
|
||||
t.Fatal("Error: object " + objectName + " not found.")
|
||||
}
|
||||
|
||||
incompObjNotFound := true
|
||||
for objIncompl := range c.ListIncompleteUploads(bucketName, objectName, isRecursive, doneCh) {
|
||||
if objIncompl.Key != "" {
|
||||
incompObjNotFound = false
|
||||
break
|
||||
}
|
||||
}
|
||||
if !incompObjNotFound {
|
||||
t.Fatal("Error: unexpected dangling incomplete upload found.")
|
||||
}
|
||||
|
||||
newReader, err := c.GetObject(bucketName, objectName)
|
||||
if err != nil {
|
||||
t.Fatal("Error: ", err)
|
||||
}
|
||||
|
@ -451,15 +681,7 @@ func TestFunctional(t *testing.T) {
|
|||
t.Fatal("Error: bytes mismatch.")
|
||||
}
|
||||
|
||||
n, err = c.FPutObject(bucketName, objectName+"-f", fileName, "text/plain")
|
||||
if err != nil {
|
||||
t.Fatal("Error: ", err)
|
||||
}
|
||||
if n != totalSize {
|
||||
t.Fatal("Error: bad length ", n, totalSize)
|
||||
}
|
||||
|
||||
err = c.FGetObject(bucketName, objectName+"-f", fileName+"-f")
|
||||
err = c.FGetObject(bucketName, objectName, fileName+"-f")
|
||||
if err != nil {
|
||||
t.Fatal("Error: ", err)
|
||||
}
|
||||
|
@ -503,7 +725,7 @@ func TestFunctional(t *testing.T) {
|
|||
t.Fatal("Error: ", err)
|
||||
}
|
||||
|
||||
newReader, _, err = c.GetObject(bucketName, objectName+"-presigned")
|
||||
newReader, err = c.GetObject(bucketName, objectName+"-presigned")
|
||||
if err != nil {
|
||||
t.Fatal("Error: ", err)
|
||||
}
|
||||
|
@ -537,11 +759,11 @@ func TestFunctional(t *testing.T) {
|
|||
if err != nil {
|
||||
t.Fatal("Error:", err)
|
||||
}
|
||||
err = c.RemoveBucket("bucket1")
|
||||
err = c.RemoveBucket(bucketName)
|
||||
if err == nil {
|
||||
t.Fatal("Error:")
|
||||
}
|
||||
if err.Error() != "The specified bucket does not exist." {
|
||||
if err.Error() != "The specified bucket does not exist" {
|
||||
t.Fatal("Error: ", err)
|
||||
}
|
||||
if err = os.Remove(fileName); err != nil {
|
|
@ -17,11 +17,126 @@
|
|||
package minio
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestSignature(t *testing.T) {
|
||||
func TestEncodeURL2Path(t *testing.T) {
|
||||
type urlStrings struct {
|
||||
objName string
|
||||
encodedObjName string
|
||||
}
|
||||
|
||||
bucketName := "bucketName"
|
||||
want := []urlStrings{
|
||||
{
|
||||
objName: "本語",
|
||||
encodedObjName: "%E6%9C%AC%E8%AA%9E",
|
||||
},
|
||||
{
|
||||
objName: "本語.1",
|
||||
encodedObjName: "%E6%9C%AC%E8%AA%9E.1",
|
||||
},
|
||||
{
|
||||
objName: ">123>3123123",
|
||||
encodedObjName: "%3E123%3E3123123",
|
||||
},
|
||||
{
|
||||
objName: "test 1 2.txt",
|
||||
encodedObjName: "test%201%202.txt",
|
||||
},
|
||||
{
|
||||
objName: "test++ 1.txt",
|
||||
encodedObjName: "test%2B%2B%201.txt",
|
||||
},
|
||||
}
|
||||
|
||||
for _, o := range want {
|
||||
u, err := url.Parse(fmt.Sprintf("https://%s.s3.amazonaws.com/%s", bucketName, o.objName))
|
||||
if err != nil {
|
||||
t.Fatal("Error:", err)
|
||||
}
|
||||
urlPath := "/" + bucketName + "/" + o.encodedObjName
|
||||
if urlPath != encodeURL2Path(u) {
|
||||
t.Fatal("Error")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestErrorResponse(t *testing.T) {
|
||||
var err error
|
||||
err = ErrorResponse{
|
||||
Code: "Testing",
|
||||
}
|
||||
errResp := ToErrorResponse(err)
|
||||
if errResp.Code != "Testing" {
|
||||
t.Fatal("Type conversion failed, we have an empty struct.")
|
||||
}
|
||||
|
||||
// Test http response decoding.
|
||||
var httpResponse *http.Response
|
||||
// Set empty variables
|
||||
httpResponse = nil
|
||||
var bucketName, objectName string
|
||||
|
||||
// Should fail with invalid argument.
|
||||
err = HTTPRespToErrorResponse(httpResponse, bucketName, objectName)
|
||||
errResp = ToErrorResponse(err)
|
||||
if errResp.Code != "InvalidArgument" {
|
||||
t.Fatal("Empty response input should return invalid argument.")
|
||||
}
|
||||
}
|
||||
|
||||
func TestSignatureCalculation(t *testing.T) {
|
||||
req, err := http.NewRequest("GET", "https://s3.amazonaws.com", nil)
|
||||
if err != nil {
|
||||
t.Fatal("Error:", err)
|
||||
}
|
||||
req = SignV4(*req, "", "", "us-east-1")
|
||||
if req.Header.Get("Authorization") != "" {
|
||||
t.Fatal("Error: anonymous credentials should not have Authorization header.")
|
||||
}
|
||||
|
||||
req = PreSignV4(*req, "", "", "us-east-1", 0)
|
||||
if strings.Contains(req.URL.RawQuery, "X-Amz-Signature") {
|
||||
t.Fatal("Error: anonymous credentials should not have Signature query resource.")
|
||||
}
|
||||
|
||||
req = SignV2(*req, "", "")
|
||||
if req.Header.Get("Authorization") != "" {
|
||||
t.Fatal("Error: anonymous credentials should not have Authorization header.")
|
||||
}
|
||||
|
||||
req = PreSignV2(*req, "", "", 0)
|
||||
if strings.Contains(req.URL.RawQuery, "Signature") {
|
||||
t.Fatal("Error: anonymous credentials should not have Signature query resource.")
|
||||
}
|
||||
|
||||
req = SignV4(*req, "ACCESS-KEY", "SECRET-KEY", "us-east-1")
|
||||
if req.Header.Get("Authorization") == "" {
|
||||
t.Fatal("Error: normal credentials should have Authorization header.")
|
||||
}
|
||||
|
||||
req = PreSignV4(*req, "ACCESS-KEY", "SECRET-KEY", "us-east-1", 0)
|
||||
if !strings.Contains(req.URL.RawQuery, "X-Amz-Signature") {
|
||||
t.Fatal("Error: normal credentials should have Signature query resource.")
|
||||
}
|
||||
|
||||
req = SignV2(*req, "ACCESS-KEY", "SECRET-KEY")
|
||||
if req.Header.Get("Authorization") == "" {
|
||||
t.Fatal("Error: normal credentials should have Authorization header.")
|
||||
}
|
||||
|
||||
req = PreSignV2(*req, "ACCESS-KEY", "SECRET-KEY", 0)
|
||||
if !strings.Contains(req.URL.RawQuery, "Signature") {
|
||||
t.Fatal("Error: normal credentials should not have Signature query resource.")
|
||||
}
|
||||
}
|
||||
|
||||
func TestSignatureType(t *testing.T) {
|
||||
clnt := Client{}
|
||||
if !clnt.signature.isV4() {
|
||||
t.Fatal("Error")
|
4
Godeps/_workspace/src/github.com/minio/minio-go/appveyor.yml
generated
vendored
4
Godeps/_workspace/src/github.com/minio/minio-go/appveyor.yml
generated
vendored
|
@ -26,8 +26,8 @@ build_script:
|
|||
- gofmt -s -l .
|
||||
- golint github.com/minio/minio-go...
|
||||
- deadcode
|
||||
- go test
|
||||
- go test -test.short -race
|
||||
- go test -short -v
|
||||
- go test -short -race -v
|
||||
|
||||
# to disable automatic tests
|
||||
test: off
|
||||
|
|
14
Godeps/_workspace/src/github.com/minio/minio-go/bucket-acl.go
generated
vendored
14
Godeps/_workspace/src/github.com/minio/minio-go/bucket-acl.go
generated
vendored
|
@ -16,7 +16,7 @@
|
|||
|
||||
package minio
|
||||
|
||||
// BucketACL - bucket level access control.
|
||||
// BucketACL - Bucket level access control.
|
||||
type BucketACL string
|
||||
|
||||
// Different types of ACL's currently supported for buckets.
|
||||
|
@ -35,7 +35,7 @@ func (b BucketACL) String() string {
|
|||
return string(b)
|
||||
}
|
||||
|
||||
// isValidBucketACL - is provided acl string supported.
|
||||
// isValidBucketACL - Is provided acl string supported.
|
||||
func (b BucketACL) isValidBucketACL() bool {
|
||||
switch true {
|
||||
case b.isPrivate():
|
||||
|
@ -47,29 +47,29 @@ func (b BucketACL) isValidBucketACL() bool {
|
|||
case b.isAuthenticated():
|
||||
return true
|
||||
case b.String() == "private":
|
||||
// by default its "private"
|
||||
// By default its "private"
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
// isPrivate - is acl Private.
|
||||
// isPrivate - Is acl Private.
|
||||
func (b BucketACL) isPrivate() bool {
|
||||
return b == bucketPrivate
|
||||
}
|
||||
|
||||
// isPublicRead - is acl PublicRead.
|
||||
// isPublicRead - Is acl PublicRead.
|
||||
func (b BucketACL) isReadOnly() bool {
|
||||
return b == bucketReadOnly
|
||||
}
|
||||
|
||||
// isPublicReadWrite - is acl PublicReadWrite.
|
||||
// isPublicReadWrite - Is acl PublicReadWrite.
|
||||
func (b BucketACL) isPublic() bool {
|
||||
return b == bucketPublic
|
||||
}
|
||||
|
||||
// isAuthenticated - is acl AuthenticatedRead.
|
||||
// isAuthenticated - Is acl AuthenticatedRead.
|
||||
func (b BucketACL) isAuthenticated() bool {
|
||||
return b == bucketAuthenticated
|
||||
}
|
||||
|
|
31
Godeps/_workspace/src/github.com/minio/minio-go/bucket-cache.go
generated
vendored
31
Godeps/_workspace/src/github.com/minio/minio-go/bucket-cache.go
generated
vendored
|
@ -24,25 +24,26 @@ import (
|
|||
"sync"
|
||||
)
|
||||
|
||||
// bucketLocationCache provides simple mechansim to hold bucket locations in memory.
|
||||
// bucketLocationCache - Provides simple mechansim to hold bucket
|
||||
// locations in memory.
|
||||
type bucketLocationCache struct {
|
||||
// Mutex is used for handling the concurrent
|
||||
// read/write requests for cache
|
||||
// mutex is used for handling the concurrent
|
||||
// read/write requests for cache.
|
||||
sync.RWMutex
|
||||
|
||||
// items holds the cached bucket locations.
|
||||
items map[string]string
|
||||
}
|
||||
|
||||
// newBucketLocationCache provides a new bucket location cache to be used
|
||||
// internally with the client object.
|
||||
// newBucketLocationCache - Provides a new bucket location cache to be
|
||||
// used internally with the client object.
|
||||
func newBucketLocationCache() *bucketLocationCache {
|
||||
return &bucketLocationCache{
|
||||
items: make(map[string]string),
|
||||
}
|
||||
}
|
||||
|
||||
// Get returns a value of a given key if it exists
|
||||
// Get - Returns a value of a given key if it exists.
|
||||
func (r *bucketLocationCache) Get(bucketName string) (location string, ok bool) {
|
||||
r.RLock()
|
||||
defer r.RUnlock()
|
||||
|
@ -50,21 +51,21 @@ func (r *bucketLocationCache) Get(bucketName string) (location string, ok bool)
|
|||
return
|
||||
}
|
||||
|
||||
// Set will persist a value to the cache
|
||||
// Set - Will persist a value into cache.
|
||||
func (r *bucketLocationCache) Set(bucketName string, location string) {
|
||||
r.Lock()
|
||||
defer r.Unlock()
|
||||
r.items[bucketName] = location
|
||||
}
|
||||
|
||||
// Delete deletes a bucket name.
|
||||
// Delete - Deletes a bucket name from cache.
|
||||
func (r *bucketLocationCache) Delete(bucketName string) {
|
||||
r.Lock()
|
||||
defer r.Unlock()
|
||||
delete(r.items, bucketName)
|
||||
}
|
||||
|
||||
// getBucketLocation - get location for the bucketName from location map cache.
|
||||
// getBucketLocation - Get location for the bucketName from location map cache.
|
||||
func (c Client) getBucketLocation(bucketName string) (string, error) {
|
||||
// For anonymous requests, default to "us-east-1" and let other calls
|
||||
// move forward.
|
||||
|
@ -101,12 +102,12 @@ func (c Client) getBucketLocation(bucketName string) (string, error) {
|
|||
}
|
||||
|
||||
location := locationConstraint
|
||||
// location is empty will be 'us-east-1'.
|
||||
// Location is empty will be 'us-east-1'.
|
||||
if location == "" {
|
||||
location = "us-east-1"
|
||||
}
|
||||
|
||||
// location can be 'EU' convert it to meaningful 'eu-west-1'.
|
||||
// Location can be 'EU' convert it to meaningful 'eu-west-1'.
|
||||
if location == "EU" {
|
||||
location = "eu-west-1"
|
||||
}
|
||||
|
@ -118,7 +119,7 @@ func (c Client) getBucketLocation(bucketName string) (string, error) {
|
|||
return location, nil
|
||||
}
|
||||
|
||||
// getBucketLocationRequest wrapper creates a new getBucketLocation request.
|
||||
// getBucketLocationRequest - Wrapper creates a new getBucketLocation request.
|
||||
func (c Client) getBucketLocationRequest(bucketName string) (*http.Request, error) {
|
||||
// Set location query.
|
||||
urlValues := make(url.Values)
|
||||
|
@ -129,16 +130,16 @@ func (c Client) getBucketLocationRequest(bucketName string) (*http.Request, erro
|
|||
targetURL.Path = filepath.Join(bucketName, "")
|
||||
targetURL.RawQuery = urlValues.Encode()
|
||||
|
||||
// get a new HTTP request for the method.
|
||||
// Get a new HTTP request for the method.
|
||||
req, err := http.NewRequest("GET", targetURL.String(), nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// set UserAgent for the request.
|
||||
// Set UserAgent for the request.
|
||||
c.setUserAgent(req)
|
||||
|
||||
// set sha256 sum for signature calculation only with signature version '4'.
|
||||
// Set sha256 sum for signature calculation only with signature version '4'.
|
||||
if c.signature.isV4() {
|
||||
req.Header.Set("X-Amz-Content-Sha256", hex.EncodeToString(sum256([]byte{})))
|
||||
}
|
||||
|
|
52
Godeps/_workspace/src/github.com/minio/minio-go/common-methods.go
generated
vendored
52
Godeps/_workspace/src/github.com/minio/minio-go/common-methods.go
generated
vendored
|
@ -1,52 +0,0 @@
|
|||
/*
|
||||
* Minio Go Library for Amazon S3 Compatible Cloud Storage (C) 2015 Minio, Inc.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package minio
|
||||
|
||||
import (
|
||||
"crypto/hmac"
|
||||
"crypto/md5"
|
||||
"crypto/sha256"
|
||||
"encoding/xml"
|
||||
"io"
|
||||
)
|
||||
|
||||
// xmlDecoder provide decoded value in xml.
|
||||
func xmlDecoder(body io.Reader, v interface{}) error {
|
||||
d := xml.NewDecoder(body)
|
||||
return d.Decode(v)
|
||||
}
|
||||
|
||||
// sum256 calculate sha256 sum for an input byte array.
|
||||
func sum256(data []byte) []byte {
|
||||
hash := sha256.New()
|
||||
hash.Write(data)
|
||||
return hash.Sum(nil)
|
||||
}
|
||||
|
||||
// sumMD5 calculate md5 sum for an input byte array.
|
||||
func sumMD5(data []byte) []byte {
|
||||
hash := md5.New()
|
||||
hash.Write(data)
|
||||
return hash.Sum(nil)
|
||||
}
|
||||
|
||||
// sumHMAC calculate hmac between two input byte array.
|
||||
func sumHMAC(key []byte, data []byte) []byte {
|
||||
hash := hmac.New(sha256.New, key)
|
||||
hash.Write(data)
|
||||
return hash.Sum(nil)
|
||||
}
|
12
Godeps/_workspace/src/github.com/minio/minio-go/constants.go
generated
vendored
12
Godeps/_workspace/src/github.com/minio/minio-go/constants.go
generated
vendored
|
@ -25,14 +25,18 @@ const minimumPartSize = 1024 * 1024 * 5
|
|||
// maxParts - maximum parts for a single multipart session.
|
||||
const maxParts = 10000
|
||||
|
||||
// maxPartSize - maximum part size 5GiB for a single multipart upload operation.
|
||||
// maxPartSize - maximum part size 5GiB for a single multipart upload
|
||||
// operation.
|
||||
const maxPartSize = 1024 * 1024 * 1024 * 5
|
||||
|
||||
// maxSinglePutObjectSize - maximum size 5GiB of object per PUT operation.
|
||||
// maxSinglePutObjectSize - maximum size 5GiB of object per PUT
|
||||
// operation.
|
||||
const maxSinglePutObjectSize = 1024 * 1024 * 1024 * 5
|
||||
|
||||
// maxMultipartPutObjectSize - maximum size 5TiB of object for Multipart operation.
|
||||
// maxMultipartPutObjectSize - maximum size 5TiB of object for
|
||||
// Multipart operation.
|
||||
const maxMultipartPutObjectSize = 1024 * 1024 * 1024 * 1024 * 5
|
||||
|
||||
// optimalReadAtBufferSize - optimal buffer 5MiB used for reading through ReadAt operation.
|
||||
// optimalReadAtBufferSize - optimal buffer 5MiB used for reading
|
||||
// through ReadAt operation.
|
||||
const optimalReadAtBufferSize = 1024 * 1024 * 5
|
||||
|
|
26
Godeps/_workspace/src/github.com/minio/minio-go/examples/play/getobject.go
generated
vendored
26
Godeps/_workspace/src/github.com/minio/minio-go/examples/play/getobject.go
generated
vendored
|
@ -29,28 +29,40 @@ import (
|
|||
func main() {
|
||||
// Note: my-bucketname, my-objectname and my-testfile are dummy values, please replace them with original values.
|
||||
|
||||
// Requests are always secure by default. set inSecure=true to enable insecure access.
|
||||
// inSecure boolean is the last argument for New().
|
||||
// Requests are always secure (HTTPS) by default. Set insecure=true to enable insecure (HTTP) access.
|
||||
// This boolean value is the last argument for New().
|
||||
|
||||
// New provides a client object backend by automatically detected signature type based
|
||||
// on the provider.
|
||||
// New returns an Amazon S3 compatible client object. API copatibality (v2 or v4) is automatically
|
||||
// determined based on the Endpoint value.
|
||||
s3Client, err := minio.New("play.minio.io:9002", "Q3AM3UQ867SPQQA43P2F", "zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG", false)
|
||||
if err != nil {
|
||||
log.Fatalln(err)
|
||||
}
|
||||
|
||||
reader, _, err := s3Client.GetObject("my-bucketname", "my-objectname")
|
||||
reader, err := s3Client.GetObject("my-bucketname", "my-objectname")
|
||||
if err != nil {
|
||||
log.Fatalln(err)
|
||||
}
|
||||
defer reader.Close()
|
||||
|
||||
localfile, err := os.Create("my-testfile")
|
||||
reader, err := s3Client.GetObject("my-bucketname", "my-objectname")
|
||||
if err != nil {
|
||||
log.Fatalln(err)
|
||||
}
|
||||
defer reader.Close()
|
||||
|
||||
localFile, err := os.Create("my-testfile")
|
||||
if err != nil {
|
||||
log.Fatalln(err)
|
||||
}
|
||||
defer localfile.Close()
|
||||
|
||||
if _, err = io.Copy(localfile, reader); err != nil {
|
||||
stat, err := reader.Stat()
|
||||
if err != nil {
|
||||
log.Fatalln(err)
|
||||
}
|
||||
|
||||
if _, err := io.CopyN(localFile, reader, stat.Size); err != nil {
|
||||
log.Fatalln(err)
|
||||
}
|
||||
}
|
||||
|
|
91
Godeps/_workspace/src/github.com/minio/minio-go/examples/play/getobjectpartial.go
generated
vendored
91
Godeps/_workspace/src/github.com/minio/minio-go/examples/play/getobjectpartial.go
generated
vendored
|
@ -1,91 +0,0 @@
|
|||
// +build ignore
|
||||
|
||||
/*
|
||||
* Minio Go Library for Amazon S3 Compatible Cloud Storage (C) 2015 Minio, Inc.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"io"
|
||||
"log"
|
||||
"os"
|
||||
|
||||
"github.com/minio/minio-go"
|
||||
)
|
||||
|
||||
func main() {
|
||||
// Note: my-bucketname, my-objectname and my-testfile are dummy values, please replace them with original values.
|
||||
|
||||
// Requests are always secure (HTTPS) by default. Set insecure=true to enable insecure (HTTP) access.
|
||||
// This boolean value is the last argument for New().
|
||||
|
||||
// New returns an Amazon S3 compatible client object. API copatibality (v2 or v4) is automatically
|
||||
// determined based on the Endpoint value.
|
||||
s3Client, err := minio.New("play.minio.io:9002", "Q3AM3UQ867SPQQA43P2F", "zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG", false)
|
||||
if err != nil {
|
||||
log.Fatalln(err)
|
||||
}
|
||||
|
||||
reader, stat, err := s3Client.GetObjectPartial("my-bucketname", "my-objectname")
|
||||
if err != nil {
|
||||
log.Fatalln(err)
|
||||
}
|
||||
defer reader.Close()
|
||||
|
||||
localFile, err := os.OpenFile("my-testfile", os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0600)
|
||||
if err != nil {
|
||||
log.Fatalln(err)
|
||||
}
|
||||
defer localfile.Close()
|
||||
|
||||
st, err := localFile.Stat()
|
||||
if err != nil {
|
||||
log.Fatalln(err)
|
||||
}
|
||||
|
||||
readAtOffset := st.Size()
|
||||
readAtBuffer := make([]byte, 5*1024*1024)
|
||||
|
||||
// Loop and write.
|
||||
for {
|
||||
readAtSize, rerr := reader.ReadAt(readAtBuffer, readAtOffset)
|
||||
if rerr != nil {
|
||||
if rerr != io.EOF {
|
||||
log.Fatalln(rerr)
|
||||
}
|
||||
}
|
||||
writeSize, werr := localFile.Write(readAtBuffer[:readAtSize])
|
||||
if werr != nil {
|
||||
log.Fatalln(werr)
|
||||
}
|
||||
if readAtSize != writeSize {
|
||||
log.Fatalln(errors.New("Something really bad happened here."))
|
||||
}
|
||||
readAtOffset += int64(writeSize)
|
||||
if rerr == io.EOF {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
// totalWritten size.
|
||||
totalWritten := readAtOffset
|
||||
|
||||
// If found mismatch error out.
|
||||
if totalWritten != stat.Size {
|
||||
log.Fatalln(errors.New("Something really bad happened here."))
|
||||
}
|
||||
}
|
3
Godeps/_workspace/src/github.com/minio/minio-go/examples/play/putobject.go
generated
vendored
3
Godeps/_workspace/src/github.com/minio/minio-go/examples/play/putobject.go
generated
vendored
|
@ -44,8 +44,7 @@ func main() {
|
|||
}
|
||||
defer object.Close()
|
||||
|
||||
st, _ := object.Stat()
|
||||
n, err := s3Client.PutObject("my-bucketname", "my-objectname", object, st.Size(), "application/octet-stream")
|
||||
n, err := s3Client.PutObject("my-bucketname", "my-objectname", object, "application/octet-stream")
|
||||
if err != nil {
|
||||
log.Fatalln(err)
|
||||
}
|
||||
|
|
56
Godeps/_workspace/src/github.com/minio/minio-go/examples/play/putobjectpartial.go
generated
vendored
56
Godeps/_workspace/src/github.com/minio/minio-go/examples/play/putobjectpartial.go
generated
vendored
|
@ -1,56 +0,0 @@
|
|||
// +build ignore
|
||||
|
||||
/*
|
||||
* Minio Go Library for Amazon S3 Compatible Cloud Storage (C) 2015 Minio, Inc.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"log"
|
||||
"os"
|
||||
|
||||
"github.com/minio/minio-go"
|
||||
)
|
||||
|
||||
func main() {
|
||||
// Note: my-bucketname, my-objectname and my-testfile are dummy values, please replace them with original values.
|
||||
|
||||
// Requests are always secure (HTTPS) by default. Set insecure=true to enable insecure (HTTP) access.
|
||||
// This boolean value is the last argument for New().
|
||||
|
||||
// New returns an Amazon S3 compatible client object. API copatibality (v2 or v4) is automatically
|
||||
// determined based on the Endpoint value.
|
||||
s3Client, err := minio.New("play.minio.io:9002", "Q3AM3UQ867SPQQA43P2F", "zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG", false)
|
||||
if err != nil {
|
||||
log.Fatalln(err)
|
||||
}
|
||||
|
||||
localFile, err := os.Open("testfile")
|
||||
if err != nil {
|
||||
log.Fatalln(err)
|
||||
}
|
||||
|
||||
st, err := localFile.Stat()
|
||||
if err != nil {
|
||||
log.Fatalln(err)
|
||||
}
|
||||
defer localFile.Close()
|
||||
|
||||
_, err = s3Client.PutObjectPartial("bucket-name", "objectName", localFile, st.Size(), "text/plain")
|
||||
if err != nil {
|
||||
log.Fatalln(err)
|
||||
}
|
||||
}
|
14
Godeps/_workspace/src/github.com/minio/minio-go/examples/s3/getobject.go
generated
vendored
14
Godeps/_workspace/src/github.com/minio/minio-go/examples/s3/getobject.go
generated
vendored
|
@ -35,23 +35,29 @@ func main() {
|
|||
|
||||
// New returns an Amazon S3 compatible client object. API copatibality (v2 or v4) is automatically
|
||||
// determined based on the Endpoint value.
|
||||
s3Client, err := minio.New("s3.amazonaws.com", "YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", false)
|
||||
s3Client, err := minio.New("s3.amazonaws.com", "YOUR-ACCESS-KEY-HERE", "YOUR-SECRET-KEY-HERE", false)
|
||||
if err != nil {
|
||||
log.Fatalln(err)
|
||||
}
|
||||
|
||||
reader, _, err := s3Client.GetObject("my-bucketname", "my-objectname")
|
||||
reader, err := s3Client.GetObject("my-bucketname", "my-objectname")
|
||||
if err != nil {
|
||||
log.Fatalln(err)
|
||||
}
|
||||
defer reader.Close()
|
||||
|
||||
localfile, err := os.Create("my-testfile")
|
||||
localFile, err := os.Create("my-testfile")
|
||||
if err != nil {
|
||||
log.Fatalln(err)
|
||||
}
|
||||
defer localfile.Close()
|
||||
|
||||
if _, err = io.Copy(localfile, reader); err != nil {
|
||||
stat, err := reader.Stat()
|
||||
if err != nil {
|
||||
log.Fatalln(err)
|
||||
}
|
||||
|
||||
if _, err := io.CopyN(localFile, reader, stat.Size); err != nil {
|
||||
log.Fatalln(err)
|
||||
}
|
||||
}
|
||||
|
|
92
Godeps/_workspace/src/github.com/minio/minio-go/examples/s3/getobjectpartial.go
generated
vendored
92
Godeps/_workspace/src/github.com/minio/minio-go/examples/s3/getobjectpartial.go
generated
vendored
|
@ -1,92 +0,0 @@
|
|||
// +build ignore
|
||||
|
||||
/*
|
||||
* Minio Go Library for Amazon S3 Compatible Cloud Storage (C) 2015 Minio, Inc.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"io"
|
||||
"log"
|
||||
"os"
|
||||
|
||||
"github.com/minio/minio-go"
|
||||
)
|
||||
|
||||
func main() {
|
||||
// Note: YOUR-ACCESSKEYID, YOUR-SECRETACCESSKEY, my-bucketname, my-objectname and
|
||||
// my-testfile are dummy values, please replace them with original values.
|
||||
|
||||
// Requests are always secure (HTTPS) by default. Set insecure=true to enable insecure (HTTP) access.
|
||||
// This boolean value is the last argument for New().
|
||||
|
||||
// New returns an Amazon S3 compatible client object. API copatibality (v2 or v4) is automatically
|
||||
// determined based on the Endpoint value.
|
||||
s3Client, err := minio.New("s3.amazonaws.com", "YOUR-ACCESS-KEY-HERE", "YOUR-SECRET-KEY-HERE", false)
|
||||
if err != nil {
|
||||
log.Fatalln(err)
|
||||
}
|
||||
|
||||
reader, stat, err := s3Client.GetObjectPartial("my-bucketname", "my-objectname")
|
||||
if err != nil {
|
||||
log.Fatalln(err)
|
||||
}
|
||||
defer reader.Close()
|
||||
|
||||
localFile, err := os.OpenFile("my-testfile", os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0600)
|
||||
if err != nil {
|
||||
log.Fatalln(err)
|
||||
}
|
||||
defer localfile.Close()
|
||||
|
||||
st, err := localFile.Stat()
|
||||
if err != nil {
|
||||
log.Fatalln(err)
|
||||
}
|
||||
|
||||
readAtOffset := st.Size()
|
||||
readAtBuffer := make([]byte, 5*1024*1024)
|
||||
|
||||
// For loop.
|
||||
for {
|
||||
readAtSize, rerr := reader.ReadAt(readAtBuffer, readAtOffset)
|
||||
if rerr != nil {
|
||||
if rerr != io.EOF {
|
||||
log.Fatalln(rerr)
|
||||
}
|
||||
}
|
||||
writeSize, werr := localFile.Write(readAtBuffer[:readAtSize])
|
||||
if werr != nil {
|
||||
log.Fatalln(werr)
|
||||
}
|
||||
if readAtSize != writeSize {
|
||||
log.Fatalln(errors.New("Something really bad happened here."))
|
||||
}
|
||||
readAtOffset += int64(writeSize)
|
||||
if rerr == io.EOF {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
// totalWritten size.
|
||||
totalWritten := readAtOffset
|
||||
|
||||
// If found mismatch error out.
|
||||
if totalWritten != stat.Size {
|
||||
log.Fatalln(errors.New("Something really bad happened here."))
|
||||
}
|
||||
}
|
3
Godeps/_workspace/src/github.com/minio/minio-go/examples/s3/putobject.go
generated
vendored
3
Godeps/_workspace/src/github.com/minio/minio-go/examples/s3/putobject.go
generated
vendored
|
@ -45,8 +45,7 @@ func main() {
|
|||
}
|
||||
defer object.Close()
|
||||
|
||||
st, _ := object.Stat()
|
||||
n, err := s3Client.PutObject("my-bucketname", "my-objectname", object, st.Size(), "application/octet-stream")
|
||||
n, err := s3Client.PutObject("my-bucketname", "my-objectname", object, "application/octet-stream")
|
||||
if err != nil {
|
||||
log.Fatalln(err)
|
||||
}
|
||||
|
|
57
Godeps/_workspace/src/github.com/minio/minio-go/examples/s3/putobjectpartial.go
generated
vendored
57
Godeps/_workspace/src/github.com/minio/minio-go/examples/s3/putobjectpartial.go
generated
vendored
|
@ -1,57 +0,0 @@
|
|||
// +build ignore
|
||||
|
||||
/*
|
||||
* Minio Go Library for Amazon S3 Compatible Cloud Storage (C) 2015 Minio, Inc.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"log"
|
||||
"os"
|
||||
|
||||
"github.com/minio/minio-go"
|
||||
)
|
||||
|
||||
func main() {
|
||||
// Note: YOUR-ACCESSKEYID, YOUR-SECRETACCESSKEY, my-bucketname, my-objectname and
|
||||
// my-testfile are dummy values, please replace them with original values.
|
||||
|
||||
// Requests are always secure (HTTPS) by default. Set insecure=true to enable insecure (HTTP) access.
|
||||
// This boolean value is the last argument for New().
|
||||
|
||||
// New returns an Amazon S3 compatible client object. API copatibality (v2 or v4) is automatically
|
||||
// determined based on the Endpoint value.
|
||||
s3Client, err := minio.New("s3.amazonaws.com", "YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", false)
|
||||
if err != nil {
|
||||
log.Fatalln(err)
|
||||
}
|
||||
|
||||
localFile, err := os.Open("my-testfile")
|
||||
if err != nil {
|
||||
log.Fatalln(err)
|
||||
}
|
||||
|
||||
st, err := localFile.Stat()
|
||||
if err != nil {
|
||||
log.Fatalln(err)
|
||||
}
|
||||
defer localFile.Close()
|
||||
|
||||
_, err = s3Client.PutObjectPartial("my-bucketname", "my-objectname", localFile, st.Size(), "text/plain")
|
||||
if err != nil {
|
||||
log.Fatalln(err)
|
||||
}
|
||||
}
|
57
Godeps/_workspace/src/github.com/minio/minio-go/post-policy.go
generated
vendored
57
Godeps/_workspace/src/github.com/minio/minio-go/post-policy.go
generated
vendored
|
@ -2,7 +2,6 @@ package minio
|
|||
|
||||
import (
|
||||
"encoding/base64"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
@ -11,7 +10,8 @@ import (
|
|||
// expirationDateFormat date format for expiration key in json policy.
|
||||
const expirationDateFormat = "2006-01-02T15:04:05.999Z"
|
||||
|
||||
// policyCondition explanation: http://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-HTTPPOSTConstructPolicy.html
|
||||
// policyCondition explanation:
|
||||
// http://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-HTTPPOSTConstructPolicy.html
|
||||
//
|
||||
// Example:
|
||||
//
|
||||
|
@ -27,11 +27,15 @@ type policyCondition struct {
|
|||
value string
|
||||
}
|
||||
|
||||
// PostPolicy provides strict static type conversion and validation for Amazon S3's POST policy JSON string.
|
||||
// PostPolicy - Provides strict static type conversion and validation
|
||||
// for Amazon S3's POST policy JSON string.
|
||||
type PostPolicy struct {
|
||||
expiration time.Time // expiration date and time of the POST policy.
|
||||
conditions []policyCondition // collection of different policy conditions.
|
||||
// contentLengthRange minimum and maximum allowable size for the uploaded content.
|
||||
// Expiration date and time of the POST policy.
|
||||
expiration time.Time
|
||||
// Collection of different policy conditions.
|
||||
conditions []policyCondition
|
||||
// ContentLengthRange minimum and maximum allowable size for the
|
||||
// uploaded content.
|
||||
contentLengthRange struct {
|
||||
min int64
|
||||
max int64
|
||||
|
@ -41,7 +45,7 @@ type PostPolicy struct {
|
|||
formData map[string]string
|
||||
}
|
||||
|
||||
// NewPostPolicy instantiate new post policy.
|
||||
// NewPostPolicy - Instantiate new post policy.
|
||||
func NewPostPolicy() *PostPolicy {
|
||||
p := &PostPolicy{}
|
||||
p.conditions = make([]policyCondition, 0)
|
||||
|
@ -49,19 +53,19 @@ func NewPostPolicy() *PostPolicy {
|
|||
return p
|
||||
}
|
||||
|
||||
// SetExpires expiration time.
|
||||
// SetExpires - Sets expiration time for the new policy.
|
||||
func (p *PostPolicy) SetExpires(t time.Time) error {
|
||||
if t.IsZero() {
|
||||
return errors.New("No expiry time set.")
|
||||
return ErrInvalidArgument("No expiry time set.")
|
||||
}
|
||||
p.expiration = t
|
||||
return nil
|
||||
}
|
||||
|
||||
// SetKey Object name.
|
||||
// SetKey - Sets an object name for the policy based upload.
|
||||
func (p *PostPolicy) SetKey(key string) error {
|
||||
if strings.TrimSpace(key) == "" || key == "" {
|
||||
return errors.New("Object name is not specified.")
|
||||
return ErrInvalidArgument("Object name is empty.")
|
||||
}
|
||||
policyCond := policyCondition{
|
||||
matchType: "eq",
|
||||
|
@ -75,10 +79,11 @@ func (p *PostPolicy) SetKey(key string) error {
|
|||
return nil
|
||||
}
|
||||
|
||||
// SetKeyStartsWith Object name that can start with.
|
||||
// SetKeyStartsWith - Sets an object name that an policy based upload
|
||||
// can start with.
|
||||
func (p *PostPolicy) SetKeyStartsWith(keyStartsWith string) error {
|
||||
if strings.TrimSpace(keyStartsWith) == "" || keyStartsWith == "" {
|
||||
return errors.New("Object prefix is not specified.")
|
||||
return ErrInvalidArgument("Object prefix is empty.")
|
||||
}
|
||||
policyCond := policyCondition{
|
||||
matchType: "starts-with",
|
||||
|
@ -92,10 +97,10 @@ func (p *PostPolicy) SetKeyStartsWith(keyStartsWith string) error {
|
|||
return nil
|
||||
}
|
||||
|
||||
// SetBucket bucket name.
|
||||
// SetBucket - Sets bucket at which objects will be uploaded to.
|
||||
func (p *PostPolicy) SetBucket(bucketName string) error {
|
||||
if strings.TrimSpace(bucketName) == "" || bucketName == "" {
|
||||
return errors.New("Bucket name is not specified.")
|
||||
return ErrInvalidArgument("Bucket name is empty.")
|
||||
}
|
||||
policyCond := policyCondition{
|
||||
matchType: "eq",
|
||||
|
@ -109,10 +114,11 @@ func (p *PostPolicy) SetBucket(bucketName string) error {
|
|||
return nil
|
||||
}
|
||||
|
||||
// SetContentType content-type.
|
||||
// SetContentType - Sets content-type of the object for this policy
|
||||
// based upload.
|
||||
func (p *PostPolicy) SetContentType(contentType string) error {
|
||||
if strings.TrimSpace(contentType) == "" || contentType == "" {
|
||||
return errors.New("No content type specified.")
|
||||
return ErrInvalidArgument("No content type specified.")
|
||||
}
|
||||
policyCond := policyCondition{
|
||||
matchType: "eq",
|
||||
|
@ -126,16 +132,17 @@ func (p *PostPolicy) SetContentType(contentType string) error {
|
|||
return nil
|
||||
}
|
||||
|
||||
// SetContentLengthRange - set new min and max content length condition.
|
||||
// SetContentLengthRange - Set new min and max content length
|
||||
// condition for all incoming uploads.
|
||||
func (p *PostPolicy) SetContentLengthRange(min, max int64) error {
|
||||
if min > max {
|
||||
return errors.New("minimum limit is larger than maximum limit")
|
||||
return ErrInvalidArgument("Minimum limit is larger than maximum limit.")
|
||||
}
|
||||
if min < 0 {
|
||||
return errors.New("minimum limit cannot be negative")
|
||||
return ErrInvalidArgument("Minimum limit cannot be negative.")
|
||||
}
|
||||
if max < 0 {
|
||||
return errors.New("maximum limit cannot be negative")
|
||||
return ErrInvalidArgument("Maximum limit cannot be negative.")
|
||||
}
|
||||
p.contentLengthRange.min = min
|
||||
p.contentLengthRange.max = max
|
||||
|
@ -145,18 +152,18 @@ func (p *PostPolicy) SetContentLengthRange(min, max int64) error {
|
|||
// addNewPolicy - internal helper to validate adding new policies.
|
||||
func (p *PostPolicy) addNewPolicy(policyCond policyCondition) error {
|
||||
if policyCond.matchType == "" || policyCond.condition == "" || policyCond.value == "" {
|
||||
return errors.New("Policy fields empty.")
|
||||
return ErrInvalidArgument("Policy fields are empty.")
|
||||
}
|
||||
p.conditions = append(p.conditions, policyCond)
|
||||
return nil
|
||||
}
|
||||
|
||||
// Stringer interface for printing in pretty manner.
|
||||
// Stringer interface for printing policy in json formatted string.
|
||||
func (p PostPolicy) String() string {
|
||||
return string(p.marshalJSON())
|
||||
}
|
||||
|
||||
// marshalJSON provides Marshalled JSON.
|
||||
// marshalJSON - Provides Marshalled JSON in bytes.
|
||||
func (p PostPolicy) marshalJSON() []byte {
|
||||
expirationStr := `"expiration":"` + p.expiration.Format(expirationDateFormat) + `"`
|
||||
var conditionsStr string
|
||||
|
@ -178,7 +185,7 @@ func (p PostPolicy) marshalJSON() []byte {
|
|||
return []byte(retStr)
|
||||
}
|
||||
|
||||
// base64 produces base64 of PostPolicy's Marshalled json.
|
||||
// base64 - Produces base64 of PostPolicy's Marshalled json.
|
||||
func (p PostPolicy) base64() string {
|
||||
return base64.StdEncoding.EncodeToString(p.marshalJSON())
|
||||
}
|
||||
|
|
40
Godeps/_workspace/src/github.com/minio/minio-go/request-signature-v2.go
generated
vendored
40
Godeps/_workspace/src/github.com/minio/minio-go/request-signature-v2.go
generated
vendored
|
@ -30,7 +30,7 @@ import (
|
|||
"time"
|
||||
)
|
||||
|
||||
// signature and API related constants.
|
||||
// Signature and API related constants.
|
||||
const (
|
||||
signV2Algorithm = "AWS"
|
||||
)
|
||||
|
@ -55,14 +55,14 @@ func encodeURL2Path(u *url.URL) (path string) {
|
|||
}
|
||||
|
||||
// PreSignV2 - presign the request in following style.
|
||||
// https://${S3_BUCKET}.s3.amazonaws.com/${S3_OBJECT}?AWSAccessKeyId=${S3_ACCESS_KEY}&Expires=${TIMESTAMP}&Signature=${SIGNATURE}
|
||||
// https://${S3_BUCKET}.s3.amazonaws.com/${S3_OBJECT}?AWSAccessKeyId=${S3_ACCESS_KEY}&Expires=${TIMESTAMP}&Signature=${SIGNATURE}.
|
||||
func PreSignV2(req http.Request, accessKeyID, secretAccessKey string, expires int64) *http.Request {
|
||||
// presign is a noop for anonymous credentials.
|
||||
// Presign is not needed for anonymous credentials.
|
||||
if accessKeyID == "" || secretAccessKey == "" {
|
||||
return nil
|
||||
return &req
|
||||
}
|
||||
d := time.Now().UTC()
|
||||
// Add date if not present
|
||||
// Add date if not present.
|
||||
if date := req.Header.Get("Date"); date == "" {
|
||||
req.Header.Set("Date", d.Format(http.TimeFormat))
|
||||
}
|
||||
|
@ -73,12 +73,12 @@ func PreSignV2(req http.Request, accessKeyID, secretAccessKey string, expires in
|
|||
// Find epoch expires when the request will expire.
|
||||
epochExpires := d.Unix() + expires
|
||||
|
||||
// get string to sign.
|
||||
// Get string to sign.
|
||||
stringToSign := fmt.Sprintf("%s\n\n\n%d\n%s", req.Method, epochExpires, path)
|
||||
hm := hmac.New(sha1.New, []byte(secretAccessKey))
|
||||
hm.Write([]byte(stringToSign))
|
||||
|
||||
// calculate signature.
|
||||
// Calculate signature.
|
||||
signature := base64.StdEncoding.EncodeToString(hm.Sum(nil))
|
||||
|
||||
query := req.URL.Query()
|
||||
|
@ -98,7 +98,8 @@ func PreSignV2(req http.Request, accessKeyID, secretAccessKey string, expires in
|
|||
return &req
|
||||
}
|
||||
|
||||
// PostPresignSignatureV2 - presigned signature for PostPolicy request
|
||||
// PostPresignSignatureV2 - presigned signature for PostPolicy
|
||||
// request.
|
||||
func PostPresignSignatureV2(policyBase64, secretAccessKey string) string {
|
||||
hm := hmac.New(sha1.New, []byte(secretAccessKey))
|
||||
hm.Write([]byte(policyBase64))
|
||||
|
@ -124,6 +125,11 @@ func PostPresignSignatureV2(policyBase64, secretAccessKey string) string {
|
|||
|
||||
// SignV2 sign the request before Do() (AWS Signature Version 2).
|
||||
func SignV2(req http.Request, accessKeyID, secretAccessKey string) *http.Request {
|
||||
// Signature calculation is not needed for anonymous credentials.
|
||||
if accessKeyID == "" || secretAccessKey == "" {
|
||||
return &req
|
||||
}
|
||||
|
||||
// Initial time.
|
||||
d := time.Now().UTC()
|
||||
|
||||
|
@ -160,11 +166,11 @@ func SignV2(req http.Request, accessKeyID, secretAccessKey string) *http.Request
|
|||
// CanonicalizedResource;
|
||||
func getStringToSignV2(req http.Request) string {
|
||||
buf := new(bytes.Buffer)
|
||||
// write standard headers.
|
||||
// Write standard headers.
|
||||
writeDefaultHeaders(buf, req)
|
||||
// write canonicalized protocol headers if any.
|
||||
// Write canonicalized protocol headers if any.
|
||||
writeCanonicalizedHeaders(buf, req)
|
||||
// write canonicalized Query resources if any.
|
||||
// Write canonicalized Query resources if any.
|
||||
writeCanonicalizedResource(buf, req)
|
||||
return buf.String()
|
||||
}
|
||||
|
@ -186,7 +192,7 @@ func writeCanonicalizedHeaders(buf *bytes.Buffer, req http.Request) {
|
|||
var protoHeaders []string
|
||||
vals := make(map[string][]string)
|
||||
for k, vv := range req.Header {
|
||||
// all the AMZ and GOOG headers should be lowercase
|
||||
// All the AMZ headers should be lowercase
|
||||
lk := strings.ToLower(k)
|
||||
if strings.HasPrefix(lk, "x-amz") {
|
||||
protoHeaders = append(protoHeaders, lk)
|
||||
|
@ -246,6 +252,7 @@ var resourceList = []string{
|
|||
// <HTTP-Request-URI, from the protocol name up to the query string> +
|
||||
// [ sub-resource, if present. For example "?acl", "?location", "?logging", or "?torrent"];
|
||||
func writeCanonicalizedResource(buf *bytes.Buffer, req http.Request) error {
|
||||
// Save request URL.
|
||||
requestURL := req.URL
|
||||
|
||||
// Get encoded URL path.
|
||||
|
@ -256,20 +263,21 @@ func writeCanonicalizedResource(buf *bytes.Buffer, req http.Request) error {
|
|||
if requestURL.RawQuery != "" {
|
||||
var n int
|
||||
vals, _ := url.ParseQuery(requestURL.RawQuery)
|
||||
// loop through all the supported resourceList.
|
||||
// Verify if any sub resource queries are present, if yes
|
||||
// canonicallize them.
|
||||
for _, resource := range resourceList {
|
||||
if vv, ok := vals[resource]; ok && len(vv) > 0 {
|
||||
n++
|
||||
// first element
|
||||
// First element
|
||||
switch n {
|
||||
case 1:
|
||||
buf.WriteByte('?')
|
||||
// the rest
|
||||
// The rest
|
||||
default:
|
||||
buf.WriteByte('&')
|
||||
}
|
||||
buf.WriteString(resource)
|
||||
// request parameters
|
||||
// Request parameters
|
||||
if len(vv[0]) > 0 {
|
||||
buf.WriteByte('=')
|
||||
buf.WriteString(url.QueryEscape(vv[0]))
|
||||
|
|
87
Godeps/_workspace/src/github.com/minio/minio-go/request-signature-v4.go
generated
vendored
87
Godeps/_workspace/src/github.com/minio/minio-go/request-signature-v4.go
generated
vendored
|
@ -26,7 +26,7 @@ import (
|
|||
"time"
|
||||
)
|
||||
|
||||
// signature and API related constants.
|
||||
// Signature and API related constants.
|
||||
const (
|
||||
signV4Algorithm = "AWS4-HMAC-SHA256"
|
||||
iso8601DateFormat = "20060102T150405Z"
|
||||
|
@ -34,28 +34,35 @@ const (
|
|||
)
|
||||
|
||||
///
|
||||
/// Excerpts from @lsegal - https://github.com/aws/aws-sdk-js/issues/659#issuecomment-120477258.
|
||||
/// Excerpts from @lsegal -
|
||||
/// https://github.com/aws/aws-sdk-js/issues/659#issuecomment-120477258.
|
||||
///
|
||||
/// User-Agent:
|
||||
///
|
||||
/// This is ignored from signing because signing this causes problems with generating pre-signed URLs
|
||||
/// (that are executed by other agents) or when customers pass requests through proxies, which may
|
||||
/// modify the user-agent.
|
||||
/// This is ignored from signing because signing this causes
|
||||
/// problems with generating pre-signed URLs (that are executed
|
||||
/// by other agents) or when customers pass requests through
|
||||
/// proxies, which may modify the user-agent.
|
||||
///
|
||||
/// Content-Length:
|
||||
///
|
||||
/// This is ignored from signing because generating a pre-signed URL should not provide a content-length
|
||||
/// constraint, specifically when vending a S3 pre-signed PUT URL. The corollary to this is that when
|
||||
/// sending regular requests (non-pre-signed), the signature contains a checksum of the body, which
|
||||
/// implicitly validates the payload length (since changing the number of bytes would change the checksum)
|
||||
/// This is ignored from signing because generating a pre-signed
|
||||
/// URL should not provide a content-length constraint,
|
||||
/// specifically when vending a S3 pre-signed PUT URL. The
|
||||
/// corollary to this is that when sending regular requests
|
||||
/// (non-pre-signed), the signature contains a checksum of the
|
||||
/// body, which implicitly validates the payload length (since
|
||||
/// changing the number of bytes would change the checksum)
|
||||
/// and therefore this header is not valuable in the signature.
|
||||
///
|
||||
/// Content-Type:
|
||||
///
|
||||
/// Signing this header causes quite a number of problems in browser environments, where browsers
|
||||
/// like to modify and normalize the content-type header in different ways. There is more information
|
||||
/// on this in https://github.com/aws/aws-sdk-js/issues/244. Avoiding this field simplifies logic
|
||||
/// and reduces the possibility of future bugs
|
||||
/// Signing this header causes quite a number of problems in
|
||||
/// browser environments, where browsers like to modify and
|
||||
/// normalize the content-type header in different ways. There is
|
||||
/// more information on this in https://goo.gl/2E9gyy. Avoiding
|
||||
/// this field simplifies logic and reduces the possibility of
|
||||
/// future bugs.
|
||||
///
|
||||
/// Authorization:
|
||||
///
|
||||
|
@ -68,7 +75,7 @@ var ignoredHeaders = map[string]bool{
|
|||
"User-Agent": true,
|
||||
}
|
||||
|
||||
// getSigningKey hmac seed to calculate final signature
|
||||
// getSigningKey hmac seed to calculate final signature.
|
||||
func getSigningKey(secret, loc string, t time.Time) []byte {
|
||||
date := sumHMAC([]byte("AWS4"+secret), []byte(t.Format(yyyymmdd)))
|
||||
location := sumHMAC(date, []byte(loc))
|
||||
|
@ -77,12 +84,13 @@ func getSigningKey(secret, loc string, t time.Time) []byte {
|
|||
return signingKey
|
||||
}
|
||||
|
||||
// getSignature final signature in hexadecimal form
|
||||
// getSignature final signature in hexadecimal form.
|
||||
func getSignature(signingKey []byte, stringToSign string) string {
|
||||
return hex.EncodeToString(sumHMAC(signingKey, []byte(stringToSign)))
|
||||
}
|
||||
|
||||
// getScope generate a string of a specific date, an AWS region, and a service
|
||||
// getScope generate a string of a specific date, an AWS region, and a
|
||||
// service.
|
||||
func getScope(location string, t time.Time) string {
|
||||
scope := strings.Join([]string{
|
||||
t.Format(yyyymmdd),
|
||||
|
@ -93,13 +101,14 @@ func getScope(location string, t time.Time) string {
|
|||
return scope
|
||||
}
|
||||
|
||||
// getCredential generate a credential string
|
||||
// getCredential generate a credential string.
|
||||
func getCredential(accessKeyID, location string, t time.Time) string {
|
||||
scope := getScope(location, t)
|
||||
return accessKeyID + "/" + scope
|
||||
}
|
||||
|
||||
// getHashedPayload get the hexadecimal value of the SHA256 hash of the request payload
|
||||
// getHashedPayload get the hexadecimal value of the SHA256 hash of
|
||||
// the request payload.
|
||||
func getHashedPayload(req http.Request) string {
|
||||
hashedPayload := req.Header.Get("X-Amz-Content-Sha256")
|
||||
if hashedPayload == "" {
|
||||
|
@ -109,7 +118,8 @@ func getHashedPayload(req http.Request) string {
|
|||
return hashedPayload
|
||||
}
|
||||
|
||||
// getCanonicalHeaders generate a list of request headers for signature.
|
||||
// getCanonicalHeaders generate a list of request headers for
|
||||
// signature.
|
||||
func getCanonicalHeaders(req http.Request) string {
|
||||
var headers []string
|
||||
vals := make(map[string][]string)
|
||||
|
@ -124,6 +134,8 @@ func getCanonicalHeaders(req http.Request) string {
|
|||
sort.Strings(headers)
|
||||
|
||||
var buf bytes.Buffer
|
||||
// Save all the headers in canonical form <header>:<value> newline
|
||||
// separated for each header.
|
||||
for _, k := range headers {
|
||||
buf.WriteString(k)
|
||||
buf.WriteByte(':')
|
||||
|
@ -145,12 +157,13 @@ func getCanonicalHeaders(req http.Request) string {
|
|||
}
|
||||
|
||||
// getSignedHeaders generate all signed request headers.
|
||||
// i.e alphabetically sorted, semicolon-separated list of lowercase request header names
|
||||
// i.e lexically sorted, semicolon-separated list of lowercase
|
||||
// request header names.
|
||||
func getSignedHeaders(req http.Request) string {
|
||||
var headers []string
|
||||
for k := range req.Header {
|
||||
if _, ok := ignoredHeaders[http.CanonicalHeaderKey(k)]; ok {
|
||||
continue // ignored header
|
||||
continue // Ignored header found continue.
|
||||
}
|
||||
headers = append(headers, strings.ToLower(k))
|
||||
}
|
||||
|
@ -168,7 +181,6 @@ func getSignedHeaders(req http.Request) string {
|
|||
// <CanonicalHeaders>\n
|
||||
// <SignedHeaders>\n
|
||||
// <HashedPayload>
|
||||
//
|
||||
func getCanonicalRequest(req http.Request) string {
|
||||
req.URL.RawQuery = strings.Replace(req.URL.Query().Encode(), "+", "%20", -1)
|
||||
canonicalRequest := strings.Join([]string{
|
||||
|
@ -193,20 +205,21 @@ func getStringToSignV4(t time.Time, location, canonicalRequest string) string {
|
|||
// PreSignV4 presign the request, in accordance with
|
||||
// http://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-query-string-auth.html.
|
||||
func PreSignV4(req http.Request, accessKeyID, secretAccessKey, location string, expires int64) *http.Request {
|
||||
// presign is a noop for anonymous credentials.
|
||||
// Presign is not needed for anonymous credentials.
|
||||
if accessKeyID == "" || secretAccessKey == "" {
|
||||
return nil
|
||||
return &req
|
||||
}
|
||||
|
||||
// Initial time.
|
||||
t := time.Now().UTC()
|
||||
|
||||
// get credential string.
|
||||
// Get credential string.
|
||||
credential := getCredential(accessKeyID, location, t)
|
||||
|
||||
// Get all signed headers.
|
||||
signedHeaders := getSignedHeaders(req)
|
||||
|
||||
// set URL query.
|
||||
// Set URL query.
|
||||
query := req.URL.Query()
|
||||
query.Set("X-Amz-Algorithm", signV4Algorithm)
|
||||
query.Set("X-Amz-Date", t.Format(iso8601DateFormat))
|
||||
|
@ -221,10 +234,10 @@ func PreSignV4(req http.Request, accessKeyID, secretAccessKey, location string,
|
|||
// Get string to sign from canonical request.
|
||||
stringToSign := getStringToSignV4(t, location, canonicalRequest)
|
||||
|
||||
// get hmac signing key.
|
||||
// Gext hmac signing key.
|
||||
signingKey := getSigningKey(secretAccessKey, location, t)
|
||||
|
||||
// calculate signature.
|
||||
// Calculate signature.
|
||||
signature := getSignature(signingKey, stringToSign)
|
||||
|
||||
// Add signature header to RawQuery.
|
||||
|
@ -233,9 +246,12 @@ func PreSignV4(req http.Request, accessKeyID, secretAccessKey, location string,
|
|||
return &req
|
||||
}
|
||||
|
||||
// PostPresignSignatureV4 - presigned signature for PostPolicy requests.
|
||||
// PostPresignSignatureV4 - presigned signature for PostPolicy
|
||||
// requests.
|
||||
func PostPresignSignatureV4(policyBase64 string, t time.Time, secretAccessKey, location string) string {
|
||||
// Get signining key.
|
||||
signingkey := getSigningKey(secretAccessKey, location, t)
|
||||
// Calculate signature.
|
||||
signature := getSignature(signingkey, policyBase64)
|
||||
return signature
|
||||
}
|
||||
|
@ -243,6 +259,11 @@ func PostPresignSignatureV4(policyBase64 string, t time.Time, secretAccessKey, l
|
|||
// SignV4 sign the request before Do(), in accordance with
|
||||
// http://docs.aws.amazon.com/AmazonS3/latest/API/sig-v4-authenticating-requests.html.
|
||||
func SignV4(req http.Request, accessKeyID, secretAccessKey, location string) *http.Request {
|
||||
// Signature calculation is not needed for anonymous credentials.
|
||||
if accessKeyID == "" || secretAccessKey == "" {
|
||||
return &req
|
||||
}
|
||||
|
||||
// Initial time.
|
||||
t := time.Now().UTC()
|
||||
|
||||
|
@ -255,19 +276,19 @@ func SignV4(req http.Request, accessKeyID, secretAccessKey, location string) *ht
|
|||
// Get string to sign from canonical request.
|
||||
stringToSign := getStringToSignV4(t, location, canonicalRequest)
|
||||
|
||||
// get hmac signing key.
|
||||
// Get hmac signing key.
|
||||
signingKey := getSigningKey(secretAccessKey, location, t)
|
||||
|
||||
// get credential string.
|
||||
// Get credential string.
|
||||
credential := getCredential(accessKeyID, location, t)
|
||||
|
||||
// Get all signed headers.
|
||||
signedHeaders := getSignedHeaders(req)
|
||||
|
||||
// calculate signature.
|
||||
// Calculate signature.
|
||||
signature := getSignature(signingKey, stringToSign)
|
||||
|
||||
// if regular request, construct the final authorization header.
|
||||
// If regular request, construct the final authorization header.
|
||||
parts := []string{
|
||||
signV4Algorithm + " Credential=" + credential,
|
||||
"SignedHeaders=" + signedHeaders,
|
||||
|
|
16
Godeps/_workspace/src/github.com/minio/minio-go/signature-type.go
generated
vendored
16
Godeps/_workspace/src/github.com/minio/minio-go/signature-type.go
generated
vendored
|
@ -1,3 +1,19 @@
|
|||
/*
|
||||
* Minio Go Library for Amazon S3 Compatible Cloud Storage (C) 2015 Minio, Inc.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package minio
|
||||
|
||||
// SignatureType is type of Authorization requested for a given HTTP request.
|
||||
|
|
2
Godeps/_workspace/src/github.com/minio/minio-go/tempfile.go
generated
vendored
2
Godeps/_workspace/src/github.com/minio/minio-go/tempfile.go
generated
vendored
|
@ -37,7 +37,7 @@ func newTempFile(prefix string) (*tempFile, error) {
|
|||
}
|
||||
return &tempFile{
|
||||
File: file,
|
||||
mutex: new(sync.Mutex),
|
||||
mutex: &sync.Mutex{},
|
||||
}, nil
|
||||
}
|
||||
|
||||
|
|
24
Godeps/_workspace/src/github.com/minio/minio-go/utils.go
generated
vendored
24
Godeps/_workspace/src/github.com/minio/minio-go/utils.go
generated
vendored
|
@ -17,7 +17,10 @@
|
|||
package minio
|
||||
|
||||
import (
|
||||
"crypto/hmac"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/xml"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"net"
|
||||
|
@ -29,6 +32,26 @@ import (
|
|||
"unicode/utf8"
|
||||
)
|
||||
|
||||
// xmlDecoder provide decoded value in xml.
|
||||
func xmlDecoder(body io.Reader, v interface{}) error {
|
||||
d := xml.NewDecoder(body)
|
||||
return d.Decode(v)
|
||||
}
|
||||
|
||||
// sum256 calculate sha256 sum for an input byte array.
|
||||
func sum256(data []byte) []byte {
|
||||
hash := sha256.New()
|
||||
hash.Write(data)
|
||||
return hash.Sum(nil)
|
||||
}
|
||||
|
||||
// sumHMAC calculate hmac between two input byte array.
|
||||
func sumHMAC(key []byte, data []byte) []byte {
|
||||
hash := hmac.New(sha256.New, key)
|
||||
hash.Write(data)
|
||||
return hash.Sum(nil)
|
||||
}
|
||||
|
||||
// isPartUploaded - true if part is already uploaded.
|
||||
func isPartUploaded(objPart objectPart, objectParts map[int]objectPart) (isUploaded bool) {
|
||||
_, isUploaded = objectParts[objPart.PartNumber]
|
||||
|
@ -261,7 +284,6 @@ func isValidObjectPrefix(objectPrefix string) error {
|
|||
// - if input object size is -1 then return maxPartSize.
|
||||
// - if it happens to be that partSize is indeed bigger
|
||||
// than the maximum part size just return maxPartSize.
|
||||
//
|
||||
func optimalPartSize(objectSize int64) int64 {
|
||||
// if object size is -1 choose part size as 5GiB.
|
||||
if objectSize == -1 {
|
||||
|
|
Loading…
Reference in a new issue