From 724b5bf4fe70910691c8da8723ff84031360ccee Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Mon, 17 Jul 2017 20:19:04 +0200 Subject: [PATCH] Update minio-go --- vendor/manifest | 2 +- .../minio/minio-go/api-compose-object.go | 532 +++ .../minio/minio-go/api-compose-object_test.go | 88 + .../minio/minio-go/api-get-object.go | 18 +- .../minio/minio-go/api-put-object-common.go | 75 - .../minio/minio-go/api-put-object-copy.go | 56 +- .../minio-go/api-put-object-encrypted.go | 46 + .../minio/minio-go/api-put-object-file.go | 201 +- .../minio-go/api-put-object-multipart.go | 211 +- .../minio/minio-go/api-put-object-progress.go | 191 - .../minio/minio-go/api-put-object-readat.go | 219 - .../minio-go/api-put-object-streaming.go | 436 ++ .../minio/minio-go/api-put-object.go | 206 +- .../src/github.com/minio/minio-go/api-stat.go | 10 +- vendor/src/github.com/minio/minio-go/api.go | 4 +- .../minio/minio-go/api_functional_v2_test.go | 1108 ----- .../minio/minio-go/api_functional_v4_test.go | 2385 ---------- .../minio/minio-go/api_unit_test.go | 21 - .../github.com/minio/minio-go/bucket-cache.go | 30 +- .../github.com/minio/minio-go/constants.go | 14 +- .../minio/minio-go/copy-conditions.go | 99 - vendor/src/github.com/minio/minio-go/core.go | 12 +- .../github.com/minio/minio-go/core_test.go | 149 +- .../src/github.com/minio/minio-go/docs/API.md | 209 +- .../minio-go/examples/s3/composeobject.go | 77 + .../minio/minio-go/examples/s3/copyobject.go | 21 +- .../examples/s3/putobject-progress.go | 4 +- .../examples/s3/putobject-s3-accelerate.go | 2 +- .../minio/minio-go/functional_tests.go | 4058 +++++++++++++++++ .../s3signer/request-signature-streaming.go | 44 +- .../src/github.com/minio/minio-go/tempfile.go | 60 - .../minio/minio-go/test-utils_test.go | 4 +- .../github.com/minio/minio-go/transport.go | 48 + .../minio/minio-go/transport_1_5.go | 39 + .../minio/minio-go/transport_1_6.go | 40 + vendor/src/github.com/minio/minio-go/utils.go | 2 +- .../github.com/minio/minio-go/utils_test.go | 5 +- 37 files changed, 5917 insertions(+), 4809 deletions(-) create mode 100644 vendor/src/github.com/minio/minio-go/api-compose-object.go create mode 100644 vendor/src/github.com/minio/minio-go/api-compose-object_test.go create mode 100644 vendor/src/github.com/minio/minio-go/api-put-object-encrypted.go delete mode 100644 vendor/src/github.com/minio/minio-go/api-put-object-progress.go delete mode 100644 vendor/src/github.com/minio/minio-go/api-put-object-readat.go create mode 100644 vendor/src/github.com/minio/minio-go/api-put-object-streaming.go delete mode 100644 vendor/src/github.com/minio/minio-go/api_functional_v2_test.go delete mode 100644 vendor/src/github.com/minio/minio-go/api_functional_v4_test.go delete mode 100644 vendor/src/github.com/minio/minio-go/copy-conditions.go create mode 100644 vendor/src/github.com/minio/minio-go/examples/s3/composeobject.go create mode 100644 vendor/src/github.com/minio/minio-go/functional_tests.go delete mode 100644 vendor/src/github.com/minio/minio-go/tempfile.go create mode 100644 vendor/src/github.com/minio/minio-go/transport.go create mode 100644 vendor/src/github.com/minio/minio-go/transport_1_5.go create mode 100644 vendor/src/github.com/minio/minio-go/transport_1_6.go diff --git a/vendor/manifest b/vendor/manifest index d0ff9d7e8..9f8f9308c 100644 --- a/vendor/manifest +++ b/vendor/manifest @@ -46,7 +46,7 @@ { "importpath": "github.com/minio/minio-go", "repository": "https://github.com/minio/minio-go", - "revision": "fe53a65ebc43b5d22626b29a19a3de81170e42d3", + "revision": "bd8e1d8a93f006a0207e026353bf0644ffcdd320", "branch": "master" }, { diff --git a/vendor/src/github.com/minio/minio-go/api-compose-object.go b/vendor/src/github.com/minio/minio-go/api-compose-object.go new file mode 100644 index 000000000..6baf09e84 --- /dev/null +++ b/vendor/src/github.com/minio/minio-go/api-compose-object.go @@ -0,0 +1,532 @@ +/* + * Minio Go Library for Amazon S3 Compatible Cloud Storage (C) 2017 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package minio + +import ( + "encoding/base64" + "fmt" + "net/http" + "net/url" + "strconv" + "strings" + "time" + + "github.com/minio/minio-go/pkg/s3utils" +) + +// SSEInfo - represents Server-Side-Encryption parameters specified by +// a user. +type SSEInfo struct { + key []byte + algo string +} + +// NewSSEInfo - specifies (binary or un-encoded) encryption key and +// algorithm name. If algo is empty, it defaults to "AES256". Ref: +// https://docs.aws.amazon.com/AmazonS3/latest/dev/ServerSideEncryptionCustomerKeys.html +func NewSSEInfo(key []byte, algo string) SSEInfo { + if algo == "" { + algo = "AES256" + } + return SSEInfo{key, algo} +} + +// internal method that computes SSE-C headers +func (s *SSEInfo) getSSEHeaders(isCopySource bool) map[string]string { + if s == nil { + return nil + } + + cs := "" + if isCopySource { + cs = "copy-source-" + } + return map[string]string{ + "x-amz-" + cs + "server-side-encryption-customer-algorithm": s.algo, + "x-amz-" + cs + "server-side-encryption-customer-key": base64.StdEncoding.EncodeToString(s.key), + "x-amz-" + cs + "server-side-encryption-customer-key-MD5": base64.StdEncoding.EncodeToString(sumMD5(s.key)), + } +} + +// GetSSEHeaders - computes and returns headers for SSE-C as key-value +// pairs. They can be set as metadata in PutObject* requests (for +// encryption) or be set as request headers in `Core.GetObject` (for +// decryption). +func (s *SSEInfo) GetSSEHeaders() map[string]string { + return s.getSSEHeaders(false) +} + +// DestinationInfo - type with information about the object to be +// created via server-side copy requests, using the Compose API. +type DestinationInfo struct { + bucket, object string + + // key for encrypting destination + encryption *SSEInfo + + // if no user-metadata is provided, it is copied from source + // (when there is only once source object in the compose + // request) + userMetadata map[string]string +} + +// NewDestinationInfo - creates a compose-object/copy-source +// destination info object. +// +// `encSSEC` is the key info for server-side-encryption with customer +// provided key. If it is nil, no encryption is performed. +// +// `userMeta` is the user-metadata key-value pairs to be set on the +// destination. The keys are automatically prefixed with `x-amz-meta-` +// if needed. If nil is passed, and if only a single source (of any +// size) is provided in the ComposeObject call, then metadata from the +// source is copied to the destination. +func NewDestinationInfo(bucket, object string, encryptSSEC *SSEInfo, + userMeta map[string]string) (d DestinationInfo, err error) { + + // Input validation. + if err = s3utils.CheckValidBucketName(bucket); err != nil { + return d, err + } + if err = s3utils.CheckValidObjectName(object); err != nil { + return d, err + } + + // Process custom-metadata to remove a `x-amz-meta-` prefix if + // present and validate that keys are distinct (after this + // prefix removal). + m := make(map[string]string) + for k, v := range userMeta { + if strings.HasPrefix(strings.ToLower(k), "x-amz-meta-") { + k = k[len("x-amz-meta-"):] + } + if _, ok := m[k]; ok { + return d, fmt.Errorf("Cannot add both %s and x-amz-meta-%s keys as custom metadata", k, k) + } + m[k] = v + } + + return DestinationInfo{ + bucket: bucket, + object: object, + encryption: encryptSSEC, + userMetadata: m, + }, nil +} + +// getUserMetaHeadersMap - construct appropriate key-value pairs to send +// as headers from metadata map to pass into copy-object request. For +// single part copy-object (i.e. non-multipart object), enable the +// withCopyDirectiveHeader to set the `x-amz-metadata-directive` to +// `REPLACE`, so that metadata headers from the source are not copied +// over. +func (d *DestinationInfo) getUserMetaHeadersMap(withCopyDirectiveHeader bool) map[string]string { + if len(d.userMetadata) == 0 { + return nil + } + r := make(map[string]string) + if withCopyDirectiveHeader { + r["x-amz-metadata-directive"] = "REPLACE" + } + for k, v := range d.userMetadata { + r["x-amz-meta-"+k] = v + } + return r +} + +// SourceInfo - represents a source object to be copied, using +// server-side copying APIs. +type SourceInfo struct { + bucket, object string + + start, end int64 + + decryptKey *SSEInfo + // Headers to send with the upload-part-copy request involving + // this source object. + Headers http.Header +} + +// NewSourceInfo - create a compose-object/copy-object source info +// object. +// +// `decryptSSEC` is the decryption key using server-side-encryption +// with customer provided key. It may be nil if the source is not +// encrypted. +func NewSourceInfo(bucket, object string, decryptSSEC *SSEInfo) SourceInfo { + r := SourceInfo{ + bucket: bucket, + object: object, + start: -1, // range is unspecified by default + decryptKey: decryptSSEC, + Headers: make(http.Header), + } + + // Set the source header + r.Headers.Set("x-amz-copy-source", s3utils.EncodePath(bucket+"/"+object)) + + // Assemble decryption headers for upload-part-copy request + for k, v := range decryptSSEC.getSSEHeaders(true) { + r.Headers.Set(k, v) + } + + return r +} + +// SetRange - Set the start and end offset of the source object to be +// copied. If this method is not called, the whole source object is +// copied. +func (s *SourceInfo) SetRange(start, end int64) error { + if start > end || start < 0 { + return ErrInvalidArgument("start must be non-negative, and start must be at most end.") + } + // Note that 0 <= start <= end + s.start, s.end = start, end + return nil +} + +// SetMatchETagCond - Set ETag match condition. The object is copied +// only if the etag of the source matches the value given here. +func (s *SourceInfo) SetMatchETagCond(etag string) error { + if etag == "" { + return ErrInvalidArgument("ETag cannot be empty.") + } + s.Headers.Set("x-amz-copy-source-if-match", etag) + return nil +} + +// SetMatchETagExceptCond - Set the ETag match exception +// condition. The object is copied only if the etag of the source is +// not the value given here. +func (s *SourceInfo) SetMatchETagExceptCond(etag string) error { + if etag == "" { + return ErrInvalidArgument("ETag cannot be empty.") + } + s.Headers.Set("x-amz-copy-source-if-none-match", etag) + return nil +} + +// SetModifiedSinceCond - Set the modified since condition. +func (s *SourceInfo) SetModifiedSinceCond(modTime time.Time) error { + if modTime.IsZero() { + return ErrInvalidArgument("Input time cannot be 0.") + } + s.Headers.Set("x-amz-copy-source-if-modified-since", modTime.Format(http.TimeFormat)) + return nil +} + +// SetUnmodifiedSinceCond - Set the unmodified since condition. +func (s *SourceInfo) SetUnmodifiedSinceCond(modTime time.Time) error { + if modTime.IsZero() { + return ErrInvalidArgument("Input time cannot be 0.") + } + s.Headers.Set("x-amz-copy-source-if-unmodified-since", modTime.Format(http.TimeFormat)) + return nil +} + +// Helper to fetch size and etag of an object using a StatObject call. +func (s *SourceInfo) getProps(c Client) (size int64, etag string, userMeta map[string]string, err error) { + // Get object info - need size and etag here. Also, decryption + // headers are added to the stat request if given. + var objInfo ObjectInfo + rh := NewGetReqHeaders() + for k, v := range s.decryptKey.getSSEHeaders(false) { + rh.Set(k, v) + } + objInfo, err = c.statObject(s.bucket, s.object, rh) + if err != nil { + err = fmt.Errorf("Could not stat object - %s/%s: %v", s.bucket, s.object, err) + } else { + size = objInfo.Size + etag = objInfo.ETag + userMeta = make(map[string]string) + for k, v := range objInfo.Metadata { + if strings.HasPrefix(k, "x-amz-meta-") { + if len(v) > 0 { + userMeta[k] = v[0] + } + } + } + } + return +} + +// uploadPartCopy - helper function to create a part in a multipart +// upload via an upload-part-copy request +// https://docs.aws.amazon.com/AmazonS3/latest/API/mpUploadUploadPartCopy.html +func (c Client) uploadPartCopy(bucket, object, uploadID string, partNumber int, + headers http.Header) (p CompletePart, err error) { + + // Build query parameters + urlValues := make(url.Values) + urlValues.Set("partNumber", strconv.Itoa(partNumber)) + urlValues.Set("uploadId", uploadID) + + // Send upload-part-copy request + resp, err := c.executeMethod("PUT", requestMetadata{ + bucketName: bucket, + objectName: object, + customHeader: headers, + queryValues: urlValues, + }) + defer closeResponse(resp) + if err != nil { + return p, err + } + + // Check if we got an error response. + if resp.StatusCode != http.StatusOK { + return p, httpRespToErrorResponse(resp, bucket, object) + } + + // Decode copy-part response on success. + cpObjRes := copyObjectResult{} + err = xmlDecoder(resp.Body, &cpObjRes) + if err != nil { + return p, err + } + p.PartNumber, p.ETag = partNumber, cpObjRes.ETag + return p, nil +} + +// ComposeObject - creates an object using server-side copying of +// existing objects. It takes a list of source objects (with optional +// offsets) and concatenates them into a new object using only +// server-side copying operations. +func (c Client) ComposeObject(dst DestinationInfo, srcs []SourceInfo) error { + if len(srcs) < 1 || len(srcs) > maxPartsCount { + return ErrInvalidArgument("There must be as least one and upto 10000 source objects.") + } + + srcSizes := make([]int64, len(srcs)) + var totalSize, size, totalParts int64 + var srcUserMeta map[string]string + var etag string + var err error + for i, src := range srcs { + size, etag, srcUserMeta, err = src.getProps(c) + if err != nil { + return fmt.Errorf("Could not get source props for %s/%s: %v", src.bucket, src.object, err) + } + + // Error out if client side encryption is used in this source object when + // more than one source objects are given. + if len(srcs) > 1 && src.Headers.Get("x-amz-meta-x-amz-key") != "" { + return ErrInvalidArgument( + fmt.Sprintf("Client side encryption is used in source object %s/%s", src.bucket, src.object)) + } + + // Since we did a HEAD to get size, we use the ETag + // value to make sure the object has not changed by + // the time we perform the copy. This is done, only if + // the user has not set their own ETag match + // condition. + if src.Headers.Get("x-amz-copy-source-if-match") == "" { + src.SetMatchETagCond(etag) + } + + // Check if a segment is specified, and if so, is the + // segment within object bounds? + if src.start != -1 { + // Since range is specified, + // 0 <= src.start <= src.end + // so only invalid case to check is: + if src.end >= size { + return ErrInvalidArgument( + fmt.Sprintf("SourceInfo %d has invalid segment-to-copy [%d, %d] (size is %d)", + i, src.start, src.end, size)) + } + size = src.end - src.start + 1 + } + + // Only the last source may be less than `absMinPartSize` + if size < absMinPartSize && i < len(srcs)-1 { + return ErrInvalidArgument( + fmt.Sprintf("SourceInfo %d is too small (%d) and it is not the last part", i, size)) + } + + // Is data to copy too large? + totalSize += size + if totalSize > maxMultipartPutObjectSize { + return ErrInvalidArgument(fmt.Sprintf("Cannot compose an object of size %d (> 5TiB)", totalSize)) + } + + // record source size + srcSizes[i] = size + + // calculate parts needed for current source + totalParts += partsRequired(size) + // Do we need more parts than we are allowed? + if totalParts > maxPartsCount { + return ErrInvalidArgument(fmt.Sprintf( + "Your proposed compose object requires more than %d parts", maxPartsCount)) + } + } + + // Single source object case (i.e. when only one source is + // involved, it is being copied wholly and at most 5GiB in + // size). + if totalParts == 1 && srcs[0].start == -1 && totalSize <= maxPartSize { + h := srcs[0].Headers + // Add destination encryption headers + for k, v := range dst.encryption.getSSEHeaders(false) { + h.Set(k, v) + } + + // If no user metadata is specified (and so, the + // for-loop below is not entered), metadata from the + // source is copied to the destination (due to + // single-part copy-object PUT request behaviour). + for k, v := range dst.getUserMetaHeadersMap(true) { + h.Set(k, v) + } + + // Send copy request + resp, err := c.executeMethod("PUT", requestMetadata{ + bucketName: dst.bucket, + objectName: dst.object, + customHeader: h, + }) + defer closeResponse(resp) + if err != nil { + return err + } + // Check if we got an error response. + if resp.StatusCode != http.StatusOK { + return httpRespToErrorResponse(resp, dst.bucket, dst.object) + } + + // Return nil on success. + return nil + } + + // Now, handle multipart-copy cases. + + // 1. Initiate a new multipart upload. + + // Set user-metadata on the destination object. If no + // user-metadata is specified, and there is only one source, + // (only) then metadata from source is copied. + userMeta := dst.getUserMetaHeadersMap(false) + metaMap := userMeta + if len(userMeta) == 0 && len(srcs) == 1 { + metaMap = srcUserMeta + } + metaHeaders := make(map[string][]string) + for k, v := range metaMap { + metaHeaders[k] = append(metaHeaders[k], v) + } + uploadID, err := c.newUploadID(dst.bucket, dst.object, metaHeaders) + if err != nil { + return fmt.Errorf("Error creating new upload: %v", err) + } + + // 2. Perform copy part uploads + objParts := []CompletePart{} + partIndex := 1 + for i, src := range srcs { + h := src.Headers + // Add destination encryption headers + for k, v := range dst.encryption.getSSEHeaders(false) { + h.Set(k, v) + } + + // calculate start/end indices of parts after + // splitting. + startIdx, endIdx := calculateEvenSplits(srcSizes[i], src) + for j, start := range startIdx { + end := endIdx[j] + + // Add (or reset) source range header for + // upload part copy request. + h.Set("x-amz-copy-source-range", + fmt.Sprintf("bytes=%d-%d", start, end)) + + // make upload-part-copy request + complPart, err := c.uploadPartCopy(dst.bucket, + dst.object, uploadID, partIndex, h) + if err != nil { + return fmt.Errorf("Error in upload-part-copy - %v", err) + } + objParts = append(objParts, complPart) + partIndex++ + } + } + + // 3. Make final complete-multipart request. + _, err = c.completeMultipartUpload(dst.bucket, dst.object, uploadID, + completeMultipartUpload{Parts: objParts}) + if err != nil { + err = fmt.Errorf("Error in complete-multipart request - %v", err) + } + return err +} + +// partsRequired is ceiling(size / copyPartSize) +func partsRequired(size int64) int64 { + r := size / copyPartSize + if size%copyPartSize > 0 { + r++ + } + return r +} + +// calculateEvenSplits - computes splits for a source and returns +// start and end index slices. Splits happen evenly to be sure that no +// part is less than 5MiB, as that could fail the multipart request if +// it is not the last part. +func calculateEvenSplits(size int64, src SourceInfo) (startIndex, endIndex []int64) { + if size == 0 { + return + } + + reqParts := partsRequired(size) + startIndex = make([]int64, reqParts) + endIndex = make([]int64, reqParts) + // Compute number of required parts `k`, as: + // + // k = ceiling(size / copyPartSize) + // + // Now, distribute the `size` bytes in the source into + // k parts as evenly as possible: + // + // r parts sized (q+1) bytes, and + // (k - r) parts sized q bytes, where + // + // size = q * k + r (by simple division of size by k, + // so that 0 <= r < k) + // + start := src.start + if start == -1 { + start = 0 + } + quot, rem := size/reqParts, size%reqParts + nextStart := start + for j := int64(0); j < reqParts; j++ { + curPartSize := quot + if j < rem { + curPartSize++ + } + + cStart := nextStart + cEnd := cStart + curPartSize - 1 + nextStart = cEnd + 1 + + startIndex[j], endIndex[j] = cStart, cEnd + } + return +} diff --git a/vendor/src/github.com/minio/minio-go/api-compose-object_test.go b/vendor/src/github.com/minio/minio-go/api-compose-object_test.go new file mode 100644 index 000000000..5339d2027 --- /dev/null +++ b/vendor/src/github.com/minio/minio-go/api-compose-object_test.go @@ -0,0 +1,88 @@ +/* + * Minio Go Library for Amazon S3 Compatible Cloud Storage (C) 2017 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package minio + +import ( + "reflect" + "testing" +) + +const ( + gb1 = 1024 * 1024 * 1024 + gb5 = 5 * gb1 + gb5p1 = gb5 + 1 + gb10p1 = 2*gb5 + 1 + gb10p2 = 2*gb5 + 2 +) + +func TestPartsRequired(t *testing.T) { + testCases := []struct { + size, ref int64 + }{ + {0, 0}, + {1, 1}, + {gb5, 1}, + {2 * gb5, 2}, + {gb10p1, 3}, + {gb10p2, 3}, + } + + for i, testCase := range testCases { + res := partsRequired(testCase.size) + if res != testCase.ref { + t.Errorf("Test %d - output did not match with reference results", i+1) + } + } +} + +func TestCalculateEvenSplits(t *testing.T) { + + testCases := []struct { + // input size and source object + size int64 + src SourceInfo + + // output part-indexes + starts, ends []int64 + }{ + {0, SourceInfo{start: -1}, nil, nil}, + {1, SourceInfo{start: -1}, []int64{0}, []int64{0}}, + {1, SourceInfo{start: 0}, []int64{0}, []int64{0}}, + + {gb1, SourceInfo{start: -1}, []int64{0}, []int64{gb1 - 1}}, + {gb5, SourceInfo{start: -1}, []int64{0}, []int64{gb5 - 1}}, + + // 2 part splits + {gb5p1, SourceInfo{start: -1}, []int64{0, gb5/2 + 1}, []int64{gb5 / 2, gb5}}, + {gb5p1, SourceInfo{start: -1}, []int64{0, gb5/2 + 1}, []int64{gb5 / 2, gb5}}, + + // 3 part splits + {gb10p1, SourceInfo{start: -1}, + []int64{0, gb10p1/3 + 1, 2*gb10p1/3 + 1}, + []int64{gb10p1 / 3, 2 * gb10p1 / 3, gb10p1 - 1}}, + + {gb10p2, SourceInfo{start: -1}, + []int64{0, gb10p2 / 3, 2 * gb10p2 / 3}, + []int64{gb10p2/3 - 1, 2*gb10p2/3 - 1, gb10p2 - 1}}, + } + + for i, testCase := range testCases { + resStart, resEnd := calculateEvenSplits(testCase.size, testCase.src) + if !reflect.DeepEqual(testCase.starts, resStart) || !reflect.DeepEqual(testCase.ends, resEnd) { + t.Errorf("Test %d - output did not match with reference results", i+1) + } + } +} diff --git a/vendor/src/github.com/minio/minio-go/api-get-object.go b/vendor/src/github.com/minio/minio-go/api-get-object.go index 1078d2f98..9bd784ffa 100644 --- a/vendor/src/github.com/minio/minio-go/api-get-object.go +++ b/vendor/src/github.com/minio/minio-go/api-get-object.go @@ -679,12 +679,18 @@ func (c Client) getObject(bucketName, objectName string, reqHeaders RequestHeade if contentType == "" { contentType = "application/octet-stream" } - var objectStat ObjectInfo - objectStat.ETag = md5sum - objectStat.Key = objectName - objectStat.Size = resp.ContentLength - objectStat.LastModified = date - objectStat.ContentType = contentType + + objectStat := ObjectInfo{ + ETag: md5sum, + Key: objectName, + Size: resp.ContentLength, + LastModified: date, + ContentType: contentType, + // Extract only the relevant header keys describing the object. + // following function filters out a list of standard set of keys + // which are not part of object metadata. + Metadata: extractObjMetadata(resp.Header), + } // do not close body here, caller will close return resp.Body, objectStat, nil diff --git a/vendor/src/github.com/minio/minio-go/api-put-object-common.go b/vendor/src/github.com/minio/minio-go/api-put-object-common.go index 213fc21f4..833f1fe8f 100644 --- a/vendor/src/github.com/minio/minio-go/api-put-object-common.go +++ b/vendor/src/github.com/minio/minio-go/api-put-object-common.go @@ -17,10 +17,8 @@ package minio import ( - "fmt" "hash" "io" - "io/ioutil" "math" "os" @@ -78,55 +76,6 @@ func optimalPartInfo(objectSize int64) (totalPartsCount int, partSize int64, las return totalPartsCount, partSize, lastPartSize, nil } -// hashCopyBuffer is identical to hashCopyN except that it doesn't take -// any size argument but takes a buffer argument and reader should be -// of io.ReaderAt interface. -// -// Stages reads from offsets into the buffer, if buffer is nil it is -// initialized to optimalBufferSize. -func hashCopyBuffer(hashAlgorithms map[string]hash.Hash, hashSums map[string][]byte, writer io.Writer, reader io.ReaderAt, buf []byte) (size int64, err error) { - hashWriter := writer - for _, v := range hashAlgorithms { - hashWriter = io.MultiWriter(hashWriter, v) - } - - // Buffer is nil, initialize. - if buf == nil { - buf = make([]byte, optimalReadBufferSize) - } - - // Offset to start reading from. - var readAtOffset int64 - - // Following block reads data at an offset from the input - // reader and copies data to into local temporary file. - for { - readAtSize, rerr := reader.ReadAt(buf, readAtOffset) - if rerr != nil { - if rerr != io.EOF { - return 0, rerr - } - } - writeSize, werr := hashWriter.Write(buf[:readAtSize]) - if werr != nil { - return 0, werr - } - if readAtSize != writeSize { - return 0, fmt.Errorf("Read size was not completely written to writer. wanted %d, got %d - %s", readAtSize, writeSize, reportIssue) - } - readAtOffset += int64(writeSize) - size += int64(writeSize) - if rerr == io.EOF { - break - } - } - - for k, v := range hashAlgorithms { - hashSums[k] = v.Sum(nil) - } - return size, err -} - // hashCopyN - Calculates chosen hashes up to partSize amount of bytes. func hashCopyN(hashAlgorithms map[string]hash.Hash, hashSums map[string][]byte, writer io.Writer, reader io.Reader, partSize int64) (size int64, err error) { hashWriter := writer @@ -167,27 +116,3 @@ func (c Client) newUploadID(bucketName, objectName string, metaData map[string][ } return initMultipartUploadResult.UploadID, nil } - -// computeHash - Calculates hashes for an input read Seeker. -func computeHash(hashAlgorithms map[string]hash.Hash, hashSums map[string][]byte, reader io.ReadSeeker) (size int64, err error) { - hashWriter := ioutil.Discard - for _, v := range hashAlgorithms { - hashWriter = io.MultiWriter(hashWriter, v) - } - - // If no buffer is provided, no need to allocate just use io.Copy. - size, err = io.Copy(hashWriter, reader) - if err != nil { - return 0, err - } - - // Seek back reader to the beginning location. - if _, err := reader.Seek(0, 0); err != nil { - return 0, err - } - - for k, v := range hashAlgorithms { - hashSums[k] = v.Sum(nil) - } - return size, nil -} diff --git a/vendor/src/github.com/minio/minio-go/api-put-object-copy.go b/vendor/src/github.com/minio/minio-go/api-put-object-copy.go index d9e2f1b57..32fa873d8 100644 --- a/vendor/src/github.com/minio/minio-go/api-put-object-copy.go +++ b/vendor/src/github.com/minio/minio-go/api-put-object-copy.go @@ -16,57 +16,7 @@ package minio -import ( - "net/http" - - "github.com/minio/minio-go/pkg/s3utils" -) - -// CopyObject - copy a source object into a new object with the provided name in the provided bucket -func (c Client) CopyObject(bucketName string, objectName string, objectSource string, cpCond CopyConditions) error { - // Input validation. - if err := s3utils.CheckValidBucketName(bucketName); err != nil { - return err - } - if err := s3utils.CheckValidObjectName(objectName); err != nil { - return err - } - if objectSource == "" { - return ErrInvalidArgument("Object source cannot be empty.") - } - - // customHeaders apply headers. - customHeaders := make(http.Header) - for _, cond := range cpCond.conditions { - customHeaders.Set(cond.key, cond.value) - } - - // Set copy source. - customHeaders.Set("x-amz-copy-source", s3utils.EncodePath(objectSource)) - - // Execute PUT on objectName. - resp, err := c.executeMethod("PUT", requestMetadata{ - bucketName: bucketName, - objectName: objectName, - customHeader: customHeaders, - }) - defer closeResponse(resp) - if err != nil { - return err - } - if resp != nil { - if resp.StatusCode != http.StatusOK { - return httpRespToErrorResponse(resp, bucketName, objectName) - } - } - - // Decode copy response on success. - cpObjRes := copyObjectResult{} - err = xmlDecoder(resp.Body, &cpObjRes) - if err != nil { - return err - } - - // Return nil on success. - return nil +// CopyObject - copy a source object into a new object +func (c Client) CopyObject(dst DestinationInfo, src SourceInfo) error { + return c.ComposeObject(dst, []SourceInfo{src}) } diff --git a/vendor/src/github.com/minio/minio-go/api-put-object-encrypted.go b/vendor/src/github.com/minio/minio-go/api-put-object-encrypted.go new file mode 100644 index 000000000..141b3e91c --- /dev/null +++ b/vendor/src/github.com/minio/minio-go/api-put-object-encrypted.go @@ -0,0 +1,46 @@ +/* + * Minio Go Library for Amazon S3 Compatible Cloud Storage (C) 2015 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package minio + +import ( + "io" + + "github.com/minio/minio-go/pkg/encrypt" +) + +// PutEncryptedObject - Encrypt and store object. +func (c Client) PutEncryptedObject(bucketName, objectName string, reader io.Reader, encryptMaterials encrypt.Materials, metadata map[string][]string, progress io.Reader) (n int64, err error) { + + if encryptMaterials == nil { + return 0, ErrInvalidArgument("Unable to recognize empty encryption properties") + } + + if err := encryptMaterials.SetupEncryptMode(reader); err != nil { + return 0, err + } + + if metadata == nil { + metadata = make(map[string][]string) + } + + // Set the necessary encryption headers, for future decryption. + metadata[amzHeaderIV] = []string{encryptMaterials.GetIV()} + metadata[amzHeaderKey] = []string{encryptMaterials.GetKey()} + metadata[amzHeaderMatDesc] = []string{encryptMaterials.GetDesc()} + + return c.putObjectMultipart(bucketName, objectName, encryptMaterials, -1, metadata, progress) +} diff --git a/vendor/src/github.com/minio/minio-go/api-put-object-file.go b/vendor/src/github.com/minio/minio-go/api-put-object-file.go index 0dc355ecf..81cdf5c2c 100644 --- a/vendor/src/github.com/minio/minio-go/api-put-object-file.go +++ b/vendor/src/github.com/minio/minio-go/api-put-object-file.go @@ -17,13 +17,9 @@ package minio import ( - "fmt" - "io" - "io/ioutil" "mime" "os" "path/filepath" - "sort" "github.com/minio/minio-go/pkg/s3utils" ) @@ -55,11 +51,6 @@ func (c Client) FPutObject(bucketName, objectName, filePath, contentType string) // Save the file size. fileSize := fileStat.Size() - // Check for largest object size allowed. - if fileSize > int64(maxMultipartPutObjectSize) { - return 0, ErrEntityTooLarge(fileSize, maxMultipartPutObjectSize, bucketName, objectName) - } - objMetadata := make(map[string][]string) // Set contentType based on filepath extension if not given or default @@ -71,195 +62,5 @@ func (c Client) FPutObject(bucketName, objectName, filePath, contentType string) } objMetadata["Content-Type"] = []string{contentType} - - // NOTE: Google Cloud Storage multipart Put is not compatible with Amazon S3 APIs. - if s3utils.IsGoogleEndpoint(c.endpointURL) { - // Do not compute MD5 for Google Cloud Storage. - return c.putObjectNoChecksum(bucketName, objectName, fileReader, fileSize, objMetadata, nil) - } - - // Small object upload is initiated for uploads for input data size smaller than 5MiB. - if fileSize < minPartSize && fileSize >= 0 { - return c.putObjectSingle(bucketName, objectName, fileReader, fileSize, objMetadata, nil) - } - - // Upload all large objects as multipart. - n, err = c.putObjectMultipartFromFile(bucketName, objectName, fileReader, fileSize, objMetadata, nil) - if err != nil { - errResp := ToErrorResponse(err) - // Verify if multipart functionality is not available, if not - // fall back to single PutObject operation. - if errResp.Code == "NotImplemented" { - // If size of file is greater than '5GiB' fail. - if fileSize > maxSinglePutObjectSize { - return 0, ErrEntityTooLarge(fileSize, maxSinglePutObjectSize, bucketName, objectName) - } - // Fall back to uploading as single PutObject operation. - return c.putObjectSingle(bucketName, objectName, fileReader, fileSize, objMetadata, nil) - } - return n, err - } - return n, nil -} - -// putObjectMultipartFromFile - Creates object from contents of *os.File -// -// NOTE: This function is meant to be used for readers with local -// file as in *os.File. This function effectively utilizes file -// system capabilities of reading from specific sections and not -// having to create temporary files. -func (c Client) putObjectMultipartFromFile(bucketName, objectName string, fileReader io.ReaderAt, fileSize int64, metaData map[string][]string, progress io.Reader) (int64, error) { - // Input validation. - if err := s3utils.CheckValidBucketName(bucketName); err != nil { - return 0, err - } - if err := s3utils.CheckValidObjectName(objectName); err != nil { - return 0, err - } - - // Initiate a new multipart upload. - uploadID, err := c.newUploadID(bucketName, objectName, metaData) - if err != nil { - return 0, err - } - - // Total data read and written to server. should be equal to 'size' at the end of the call. - var totalUploadedSize int64 - - // Complete multipart upload. - var complMultipartUpload completeMultipartUpload - - // Calculate the optimal parts info for a given size. - totalPartsCount, partSize, lastPartSize, err := optimalPartInfo(fileSize) - if err != nil { - return 0, err - } - - // Create a channel to communicate a part was uploaded. - // Buffer this to 10000, the maximum number of parts allowed by S3. - uploadedPartsCh := make(chan uploadedPartRes, 10000) - - // Create a channel to communicate which part to upload. - // Buffer this to 10000, the maximum number of parts allowed by S3. - uploadPartsCh := make(chan uploadPartReq, 10000) - - // Just for readability. - lastPartNumber := totalPartsCount - - // Initialize parts uploaded map. - partsInfo := make(map[int]ObjectPart) - - // Send each part through the partUploadCh to be uploaded. - for p := 1; p <= totalPartsCount; p++ { - part, ok := partsInfo[p] - if ok { - uploadPartsCh <- uploadPartReq{PartNum: p, Part: &part} - } else { - uploadPartsCh <- uploadPartReq{PartNum: p, Part: nil} - } - } - close(uploadPartsCh) - - // Use three 'workers' to upload parts in parallel. - for w := 1; w <= totalWorkers; w++ { - go func() { - // Deal with each part as it comes through the channel. - for uploadReq := range uploadPartsCh { - // Add hash algorithms that need to be calculated by computeHash() - // In case of a non-v4 signature or https connection, sha256 is not needed. - hashAlgos, hashSums := c.hashMaterials() - - // If partNumber was not uploaded we calculate the missing - // part offset and size. For all other part numbers we - // calculate offset based on multiples of partSize. - readOffset := int64(uploadReq.PartNum-1) * partSize - missingPartSize := partSize - - // As a special case if partNumber is lastPartNumber, we - // calculate the offset based on the last part size. - if uploadReq.PartNum == lastPartNumber { - readOffset = (fileSize - lastPartSize) - missingPartSize = lastPartSize - } - - // Get a section reader on a particular offset. - sectionReader := io.NewSectionReader(fileReader, readOffset, missingPartSize) - var prtSize int64 - var err error - - prtSize, err = computeHash(hashAlgos, hashSums, sectionReader) - if err != nil { - uploadedPartsCh <- uploadedPartRes{ - Error: err, - } - // Exit the goroutine. - return - } - - // Proceed to upload the part. - var objPart ObjectPart - objPart, err = c.uploadPart(bucketName, objectName, uploadID, sectionReader, uploadReq.PartNum, - hashSums["md5"], hashSums["sha256"], prtSize) - if err != nil { - uploadedPartsCh <- uploadedPartRes{ - Error: err, - } - // Exit the goroutine. - return - } - - // Save successfully uploaded part metadata. - uploadReq.Part = &objPart - - // Return through the channel the part size. - uploadedPartsCh <- uploadedPartRes{ - Size: missingPartSize, - PartNum: uploadReq.PartNum, - Part: uploadReq.Part, - Error: nil, - } - } - }() - } - - // Retrieve each uploaded part once it is done. - for u := 1; u <= totalPartsCount; u++ { - uploadRes := <-uploadedPartsCh - if uploadRes.Error != nil { - return totalUploadedSize, uploadRes.Error - } - // Retrieve each uploaded part and store it to be completed. - part := uploadRes.Part - if part == nil { - return totalUploadedSize, ErrInvalidArgument(fmt.Sprintf("Missing part number %d", uploadRes.PartNum)) - } - // Update the total uploaded size. - totalUploadedSize += uploadRes.Size - // Update the progress bar if there is one. - if progress != nil { - if _, err = io.CopyN(ioutil.Discard, progress, uploadRes.Size); err != nil { - return totalUploadedSize, err - } - } - // Store the part to be completed. - complMultipartUpload.Parts = append(complMultipartUpload.Parts, CompletePart{ - ETag: part.ETag, - PartNumber: part.PartNumber, - }) - } - - // Verify if we uploaded all data. - if totalUploadedSize != fileSize { - return totalUploadedSize, ErrUnexpectedEOF(totalUploadedSize, fileSize, bucketName, objectName) - } - - // Sort all completed parts. - sort.Sort(completedParts(complMultipartUpload.Parts)) - _, err = c.completeMultipartUpload(bucketName, objectName, uploadID, complMultipartUpload) - if err != nil { - return totalUploadedSize, err - } - - // Return final size. - return totalUploadedSize, nil + return c.putObjectCommon(bucketName, objectName, fileReader, fileSize, objMetadata, nil) } diff --git a/vendor/src/github.com/minio/minio-go/api-put-object-multipart.go b/vendor/src/github.com/minio/minio-go/api-put-object-multipart.go index 507fd65d6..1938378f8 100644 --- a/vendor/src/github.com/minio/minio-go/api-put-object-multipart.go +++ b/vendor/src/github.com/minio/minio-go/api-put-object-multipart.go @@ -24,7 +24,6 @@ import ( "io/ioutil" "net/http" "net/url" - "os" "sort" "strconv" "strings" @@ -32,161 +31,60 @@ import ( "github.com/minio/minio-go/pkg/s3utils" ) -// Comprehensive put object operation involving multipart uploads. -// -// Following code handles these types of readers. -// -// - *os.File -// - *minio.Object -// - Any reader which has a method 'ReadAt()' -// -func (c Client) putObjectMultipart(bucketName, objectName string, reader io.Reader, size int64, metaData map[string][]string, progress io.Reader) (n int64, err error) { - if size > 0 && size > minPartSize { - // Verify if reader is *os.File, then use file system functionalities. - if isFile(reader) { - return c.putObjectMultipartFromFile(bucketName, objectName, reader.(*os.File), size, metaData, progress) - } - // Verify if reader is *minio.Object or io.ReaderAt. - // NOTE: Verification of object is kept for a specific purpose - // while it is going to be duck typed similar to io.ReaderAt. - // It is to indicate that *minio.Object implements io.ReaderAt. - // and such a functionality is used in the subsequent code - // path. - if isObject(reader) || isReadAt(reader) { - return c.putObjectMultipartFromReadAt(bucketName, objectName, reader.(io.ReaderAt), size, metaData, progress) +func (c Client) putObjectMultipart(bucketName, objectName string, reader io.Reader, size int64, + metadata map[string][]string, progress io.Reader) (n int64, err error) { + n, err = c.putObjectMultipartNoStream(bucketName, objectName, reader, size, metadata, progress) + if err != nil { + errResp := ToErrorResponse(err) + // Verify if multipart functionality is not available, if not + // fall back to single PutObject operation. + if errResp.Code == "AccessDenied" && strings.Contains(errResp.Message, "Access Denied") { + // Verify if size of reader is greater than '5GiB'. + if size > maxSinglePutObjectSize { + return 0, ErrEntityTooLarge(size, maxSinglePutObjectSize, bucketName, objectName) + } + // Fall back to uploading as single PutObject operation. + return c.putObjectNoChecksum(bucketName, objectName, reader, size, metadata, progress) } } - // For any other data size and reader type we do generic multipart - // approach by staging data in temporary files and uploading them. - return c.putObjectMultipartStream(bucketName, objectName, reader, size, metaData, progress) + return n, err } -// putObjectMultipartStreamNoChecksum - upload a large object using -// multipart upload and streaming signature for signing payload. -func (c Client) putObjectMultipartStreamNoChecksum(bucketName, objectName string, - reader io.Reader, size int64, metadata map[string][]string, progress io.Reader) (int64, error) { - +func (c Client) putObjectMultipartNoStream(bucketName, objectName string, reader io.Reader, size int64, + metadata map[string][]string, progress io.Reader) (n int64, err error) { // Input validation. - if err := s3utils.CheckValidBucketName(bucketName); err != nil { + if err = s3utils.CheckValidBucketName(bucketName); err != nil { return 0, err } - if err := s3utils.CheckValidObjectName(objectName); err != nil { + if err = s3utils.CheckValidObjectName(objectName); err != nil { return 0, err } - // Initiates a new multipart request - uploadID, err := c.newUploadID(bucketName, objectName, metadata) - if err != nil { - return 0, err - } - - // Calculate the optimal parts info for a given size. - totalPartsCount, partSize, lastPartSize, err := optimalPartInfo(size) - if err != nil { - return 0, err - } - - // Total data read and written to server. should be equal to 'size' at the end of the call. - var totalUploadedSize int64 - - // Initialize parts uploaded map. - partsInfo := make(map[int]ObjectPart) - - // Part number always starts with '1'. - var partNumber int - for partNumber = 1; partNumber <= totalPartsCount; partNumber++ { - // Update progress reader appropriately to the latest offset - // as we read from the source. - hookReader := newHook(reader, progress) - - // Proceed to upload the part. - if partNumber == totalPartsCount { - partSize = lastPartSize - } - - var objPart ObjectPart - objPart, err = c.uploadPart(bucketName, objectName, uploadID, - io.LimitReader(hookReader, partSize), partNumber, nil, nil, partSize) - // For unknown size, Read EOF we break away. - // We do not have to upload till totalPartsCount. - if err == io.EOF && size < 0 { - break - } - - if err != nil { - return totalUploadedSize, err - } - - // Save successfully uploaded part metadata. - partsInfo[partNumber] = objPart - - // Save successfully uploaded size. - totalUploadedSize += partSize - } - - // Verify if we uploaded all the data. - if size > 0 { - if totalUploadedSize != size { - return totalUploadedSize, ErrUnexpectedEOF(totalUploadedSize, size, bucketName, objectName) - } - } - - // Complete multipart upload. - var complMultipartUpload completeMultipartUpload - - // Loop over total uploaded parts to save them in - // Parts array before completing the multipart request. - for i := 1; i < partNumber; i++ { - part, ok := partsInfo[i] - if !ok { - return 0, ErrInvalidArgument(fmt.Sprintf("Missing part number %d", i)) - } - complMultipartUpload.Parts = append(complMultipartUpload.Parts, CompletePart{ - ETag: part.ETag, - PartNumber: part.PartNumber, - }) - } - - // Sort all completed parts. - sort.Sort(completedParts(complMultipartUpload.Parts)) - _, err = c.completeMultipartUpload(bucketName, objectName, uploadID, complMultipartUpload) - if err != nil { - return totalUploadedSize, err - } - - // Return final size. - return totalUploadedSize, nil -} - -// putObjectStream uploads files bigger than 64MiB, and also supports -// special case where size is unknown i.e '-1'. -func (c Client) putObjectMultipartStream(bucketName, objectName string, reader io.Reader, size int64, metaData map[string][]string, progress io.Reader) (n int64, err error) { - // Input validation. - if err := s3utils.CheckValidBucketName(bucketName); err != nil { - return 0, err - } - if err := s3utils.CheckValidObjectName(objectName); err != nil { - return 0, err - } - - // Total data read and written to server. should be equal to 'size' at the end of the call. + // Total data read and written to server. should be equal to + // 'size' at the end of the call. var totalUploadedSize int64 // Complete multipart upload. var complMultipartUpload completeMultipartUpload - // Initiate a new multipart upload. - uploadID, err := c.newUploadID(bucketName, objectName, metaData) - if err != nil { - return 0, err - } - // Calculate the optimal parts info for a given size. totalPartsCount, partSize, _, err := optimalPartInfo(size) if err != nil { return 0, err } + // Initiate a new multipart upload. + uploadID, err := c.newUploadID(bucketName, objectName, metadata) + if err != nil { + return 0, err + } + + defer func() { + if err != nil { + c.abortMultipartUpload(bucketName, objectName, uploadID) + } + }() + // Part number always starts with '1'. partNumber := 1 @@ -197,8 +95,9 @@ func (c Client) putObjectMultipartStream(bucketName, objectName string, reader i partsInfo := make(map[int]ObjectPart) for partNumber <= totalPartsCount { - // Choose hash algorithms to be calculated by hashCopyN, avoid sha256 - // with non-v4 signature request or HTTPS connection + // Choose hash algorithms to be calculated by hashCopyN, + // avoid sha256 with non-v4 signature request or + // HTTPS connection. hashAlgos, hashSums := c.hashMaterials() // Calculates hash sums while copying partSize bytes into tmpBuffer. @@ -214,7 +113,8 @@ func (c Client) putObjectMultipartStream(bucketName, objectName string, reader i // Proceed to upload the part. var objPart ObjectPart - objPart, err = c.uploadPart(bucketName, objectName, uploadID, reader, partNumber, hashSums["md5"], hashSums["sha256"], prtSize) + objPart, err = c.uploadPart(bucketName, objectName, uploadID, reader, partNumber, + hashSums["md5"], hashSums["sha256"], prtSize, metadata) if err != nil { // Reset the temporary buffer upon any error. tmpBuffer.Reset() @@ -224,13 +124,6 @@ func (c Client) putObjectMultipartStream(bucketName, objectName string, reader i // Save successfully uploaded part metadata. partsInfo[partNumber] = objPart - // Update the progress reader for the skipped part. - if progress != nil { - if _, err = io.CopyN(ioutil.Discard, progress, prtSize); err != nil { - return totalUploadedSize, err - } - } - // Reset the temporary buffer. tmpBuffer.Reset() @@ -269,8 +162,7 @@ func (c Client) putObjectMultipartStream(bucketName, objectName string, reader i // Sort all completed parts. sort.Sort(completedParts(complMultipartUpload.Parts)) - _, err = c.completeMultipartUpload(bucketName, objectName, uploadID, complMultipartUpload) - if err != nil { + if _, err = c.completeMultipartUpload(bucketName, objectName, uploadID, complMultipartUpload); err != nil { return totalUploadedSize, err } @@ -279,7 +171,7 @@ func (c Client) putObjectMultipartStream(bucketName, objectName string, reader i } // initiateMultipartUpload - Initiates a multipart upload and returns an upload ID. -func (c Client) initiateMultipartUpload(bucketName, objectName string, metaData map[string][]string) (initiateMultipartUploadResult, error) { +func (c Client) initiateMultipartUpload(bucketName, objectName string, metadata map[string][]string) (initiateMultipartUploadResult, error) { // Input validation. if err := s3utils.CheckValidBucketName(bucketName); err != nil { return initiateMultipartUploadResult{}, err @@ -294,14 +186,14 @@ func (c Client) initiateMultipartUpload(bucketName, objectName string, metaData // Set ContentType header. customHeader := make(http.Header) - for k, v := range metaData { + for k, v := range metadata { if len(v) > 0 { customHeader.Set(k, v[0]) } } // Set a default content-type header if the latter is not provided - if v, ok := metaData["Content-Type"]; !ok || len(v) == 0 { + if v, ok := metadata["Content-Type"]; !ok || len(v) == 0 { customHeader.Set("Content-Type", "application/octet-stream") } @@ -332,8 +224,11 @@ func (c Client) initiateMultipartUpload(bucketName, objectName string, metaData return initiateMultipartUploadResult, nil } +const serverEncryptionKeyPrefix = "x-amz-server-side-encryption" + // uploadPart - Uploads a part in a multipart upload. -func (c Client) uploadPart(bucketName, objectName, uploadID string, reader io.Reader, partNumber int, md5Sum, sha256Sum []byte, size int64) (ObjectPart, error) { +func (c Client) uploadPart(bucketName, objectName, uploadID string, reader io.Reader, + partNumber int, md5Sum, sha256Sum []byte, size int64, metadata map[string][]string) (ObjectPart, error) { // Input validation. if err := s3utils.CheckValidBucketName(bucketName); err != nil { return ObjectPart{}, err @@ -361,10 +256,21 @@ func (c Client) uploadPart(bucketName, objectName, uploadID string, reader io.Re // Set upload id. urlValues.Set("uploadId", uploadID) + // Set encryption headers, if any. + customHeader := make(http.Header) + for k, v := range metadata { + if len(v) > 0 { + if strings.HasPrefix(strings.ToLower(k), serverEncryptionKeyPrefix) { + customHeader.Set(k, v[0]) + } + } + } + reqMetadata := requestMetadata{ bucketName: bucketName, objectName: objectName, queryValues: urlValues, + customHeader: customHeader, contentBody: reader, contentLength: size, contentMD5Bytes: md5Sum, @@ -393,7 +299,8 @@ func (c Client) uploadPart(bucketName, objectName, uploadID string, reader io.Re } // completeMultipartUpload - Completes a multipart upload by assembling previously uploaded parts. -func (c Client) completeMultipartUpload(bucketName, objectName, uploadID string, complete completeMultipartUpload) (completeMultipartUploadResult, error) { +func (c Client) completeMultipartUpload(bucketName, objectName, uploadID string, + complete completeMultipartUpload) (completeMultipartUploadResult, error) { // Input validation. if err := s3utils.CheckValidBucketName(bucketName); err != nil { return completeMultipartUploadResult{}, err diff --git a/vendor/src/github.com/minio/minio-go/api-put-object-progress.go b/vendor/src/github.com/minio/minio-go/api-put-object-progress.go deleted file mode 100644 index fc4c40ad4..000000000 --- a/vendor/src/github.com/minio/minio-go/api-put-object-progress.go +++ /dev/null @@ -1,191 +0,0 @@ -/* - * Minio Go Library for Amazon S3 Compatible Cloud Storage (C) 2015 Minio, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package minio - -import ( - "io" - "strings" - - "github.com/minio/minio-go/pkg/credentials" - "github.com/minio/minio-go/pkg/encrypt" - "github.com/minio/minio-go/pkg/s3utils" -) - -// PutObjectWithProgress - with progress. -func (c Client) PutObjectWithProgress(bucketName, objectName string, reader io.Reader, contentType string, progress io.Reader) (n int64, err error) { - metaData := make(map[string][]string) - metaData["Content-Type"] = []string{contentType} - return c.PutObjectWithMetadata(bucketName, objectName, reader, metaData, progress) -} - -// PutEncryptedObject - Encrypt and store object. -func (c Client) PutEncryptedObject(bucketName, objectName string, reader io.Reader, encryptMaterials encrypt.Materials, metaData map[string][]string, progress io.Reader) (n int64, err error) { - - if encryptMaterials == nil { - return 0, ErrInvalidArgument("Unable to recognize empty encryption properties") - } - - if err := encryptMaterials.SetupEncryptMode(reader); err != nil { - return 0, err - } - - if metaData == nil { - metaData = make(map[string][]string) - } - - // Set the necessary encryption headers, for future decryption. - metaData[amzHeaderIV] = []string{encryptMaterials.GetIV()} - metaData[amzHeaderKey] = []string{encryptMaterials.GetKey()} - metaData[amzHeaderMatDesc] = []string{encryptMaterials.GetDesc()} - - return c.PutObjectWithMetadata(bucketName, objectName, encryptMaterials, metaData, progress) -} - -// PutObjectWithMetadata - with metadata. -func (c Client) PutObjectWithMetadata(bucketName, objectName string, reader io.Reader, metaData map[string][]string, progress io.Reader) (n int64, err error) { - // Input validation. - if err := s3utils.CheckValidBucketName(bucketName); err != nil { - return 0, err - } - if err := s3utils.CheckValidObjectName(objectName); err != nil { - return 0, err - } - if reader == nil { - return 0, ErrInvalidArgument("Input reader is invalid, cannot be nil.") - } - - // Size of the object. - var size int64 - - // Get reader size. - size, err = getReaderSize(reader) - if err != nil { - return 0, err - } - - // Check for largest object size allowed. - if size > int64(maxMultipartPutObjectSize) { - return 0, ErrEntityTooLarge(size, maxMultipartPutObjectSize, bucketName, objectName) - } - - // NOTE: Google Cloud Storage does not implement Amazon S3 Compatible multipart PUT. - if s3utils.IsGoogleEndpoint(c.endpointURL) { - // Do not compute MD5 for Google Cloud Storage. - return c.putObjectNoChecksum(bucketName, objectName, reader, size, metaData, progress) - } - - // putSmall object. - if size < minPartSize && size >= 0 { - return c.putObjectSingle(bucketName, objectName, reader, size, metaData, progress) - } - - // For all sizes greater than 5MiB do multipart. - n, err = c.putObjectMultipart(bucketName, objectName, reader, size, metaData, progress) - if err != nil { - errResp := ToErrorResponse(err) - // Verify if multipart functionality is not available, if not - // fall back to single PutObject operation. - if errResp.Code == "AccessDenied" && strings.Contains(errResp.Message, "Access Denied") { - // Verify if size of reader is greater than '5GiB'. - if size > maxSinglePutObjectSize { - return 0, ErrEntityTooLarge(size, maxSinglePutObjectSize, bucketName, objectName) - } - // Fall back to uploading as single PutObject operation. - return c.putObjectSingle(bucketName, objectName, reader, size, metaData, progress) - } - return n, err - } - return n, nil -} - -// PutObjectStreaming using AWS streaming signature V4 -func (c Client) PutObjectStreaming(bucketName, objectName string, reader io.Reader) (n int64, err error) { - return c.PutObjectStreamingWithProgress(bucketName, objectName, reader, nil, nil) -} - -// PutObjectStreamingWithMetadata using AWS streaming signature V4 -func (c Client) PutObjectStreamingWithMetadata(bucketName, objectName string, reader io.Reader, metadata map[string][]string) (n int64, err error) { - return c.PutObjectStreamingWithProgress(bucketName, objectName, reader, metadata, nil) -} - -// PutObjectStreamingWithProgress using AWS streaming signature V4 -func (c Client) PutObjectStreamingWithProgress(bucketName, objectName string, reader io.Reader, metadata map[string][]string, progress io.Reader) (n int64, err error) { - // NOTE: Streaming signature is not supported by GCS. - if s3utils.IsGoogleEndpoint(c.endpointURL) { - return 0, ErrorResponse{ - Code: "NotImplemented", - Message: "AWS streaming signature v4 is not supported with Google Cloud Storage", - Key: objectName, - BucketName: bucketName, - } - } - - if c.overrideSignerType.IsV2() { - return 0, ErrorResponse{ - Code: "NotImplemented", - Message: "AWS streaming signature v4 is not supported with minio client initialized for AWS signature v2", - Key: objectName, - BucketName: bucketName, - } - } - - // Size of the object. - var size int64 - - // Get reader size. - size, err = getReaderSize(reader) - if err != nil { - return 0, err - } - - // Check for largest object size allowed. - if size > int64(maxMultipartPutObjectSize) { - return 0, ErrEntityTooLarge(size, maxMultipartPutObjectSize, bucketName, objectName) - } - - // If size cannot be found on a stream, it is not possible - // to upload using streaming signature, fall back to multipart. - if size < 0 { - return c.putObjectMultipartStream(bucketName, objectName, reader, size, metadata, progress) - } - - // Set streaming signature. - c.overrideSignerType = credentials.SignatureV4Streaming - - if size < minPartSize && size >= 0 { - return c.putObjectNoChecksum(bucketName, objectName, reader, size, metadata, progress) - } - - // For all sizes greater than 64MiB do multipart. - n, err = c.putObjectMultipartStreamNoChecksum(bucketName, objectName, reader, size, metadata, progress) - if err != nil { - errResp := ToErrorResponse(err) - // Verify if multipart functionality is not available, if not - // fall back to single PutObject operation. - if errResp.Code == "AccessDenied" && strings.Contains(errResp.Message, "Access Denied") { - // Verify if size of reader is greater than '5GiB'. - if size > maxSinglePutObjectSize { - return 0, ErrEntityTooLarge(size, maxSinglePutObjectSize, bucketName, objectName) - } - // Fall back to uploading as single PutObject operation. - return c.putObjectNoChecksum(bucketName, objectName, reader, size, metadata, progress) - } - return n, err - } - - return n, nil -} diff --git a/vendor/src/github.com/minio/minio-go/api-put-object-readat.go b/vendor/src/github.com/minio/minio-go/api-put-object-readat.go deleted file mode 100644 index 1c20f1818..000000000 --- a/vendor/src/github.com/minio/minio-go/api-put-object-readat.go +++ /dev/null @@ -1,219 +0,0 @@ -/* - * Minio Go Library for Amazon S3 Compatible Cloud Storage (C) 2015, 2016 Minio, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package minio - -import ( - "bytes" - "fmt" - "io" - "io/ioutil" - "sort" - - "github.com/minio/minio-go/pkg/s3utils" -) - -// uploadedPartRes - the response received from a part upload. -type uploadedPartRes struct { - Error error // Any error encountered while uploading the part. - PartNum int // Number of the part uploaded. - Size int64 // Size of the part uploaded. - Part *ObjectPart -} - -type uploadPartReq struct { - PartNum int // Number of the part uploaded. - Part *ObjectPart // Size of the part uploaded. -} - -// putObjectMultipartFromReadAt - Uploads files bigger than 5MiB. Supports reader -// of type which implements io.ReaderAt interface (ReadAt method). -// -// NOTE: This function is meant to be used for all readers which -// implement io.ReaderAt which allows us for resuming multipart -// uploads but reading at an offset, which would avoid re-read the -// data which was already uploaded. Internally this function uses -// temporary files for staging all the data, these temporary files are -// cleaned automatically when the caller i.e http client closes the -// stream after uploading all the contents successfully. -func (c Client) putObjectMultipartFromReadAt(bucketName, objectName string, reader io.ReaderAt, size int64, metaData map[string][]string, progress io.Reader) (n int64, err error) { - // Input validation. - if err := s3utils.CheckValidBucketName(bucketName); err != nil { - return 0, err - } - if err := s3utils.CheckValidObjectName(objectName); err != nil { - return 0, err - } - - // Initiate a new multipart upload. - uploadID, err := c.newUploadID(bucketName, objectName, metaData) - if err != nil { - return 0, err - } - - // Total data read and written to server. should be equal to 'size' at the end of the call. - var totalUploadedSize int64 - - // Complete multipart upload. - var complMultipartUpload completeMultipartUpload - - // Calculate the optimal parts info for a given size. - totalPartsCount, partSize, lastPartSize, err := optimalPartInfo(size) - if err != nil { - return 0, err - } - - // Declare a channel that sends the next part number to be uploaded. - // Buffered to 10000 because thats the maximum number of parts allowed - // by S3. - uploadPartsCh := make(chan uploadPartReq, 10000) - - // Declare a channel that sends back the response of a part upload. - // Buffered to 10000 because thats the maximum number of parts allowed - // by S3. - uploadedPartsCh := make(chan uploadedPartRes, 10000) - - // Used for readability, lastPartNumber is always totalPartsCount. - lastPartNumber := totalPartsCount - - // Initialize parts uploaded map. - partsInfo := make(map[int]ObjectPart) - - // Send each part number to the channel to be processed. - for p := 1; p <= totalPartsCount; p++ { - part, ok := partsInfo[p] - if ok { - uploadPartsCh <- uploadPartReq{PartNum: p, Part: &part} - } else { - uploadPartsCh <- uploadPartReq{PartNum: p, Part: nil} - } - } - close(uploadPartsCh) - - // Receive each part number from the channel allowing three parallel uploads. - for w := 1; w <= totalWorkers; w++ { - go func() { - // Read defaults to reading at 5MiB buffer. - readAtBuffer := make([]byte, optimalReadBufferSize) - - // Each worker will draw from the part channel and upload in parallel. - for uploadReq := range uploadPartsCh { - // Declare a new tmpBuffer. - tmpBuffer := new(bytes.Buffer) - - // If partNumber was not uploaded we calculate the missing - // part offset and size. For all other part numbers we - // calculate offset based on multiples of partSize. - readOffset := int64(uploadReq.PartNum-1) * partSize - missingPartSize := partSize - - // As a special case if partNumber is lastPartNumber, we - // calculate the offset based on the last part size. - if uploadReq.PartNum == lastPartNumber { - readOffset = (size - lastPartSize) - missingPartSize = lastPartSize - } - - // Get a section reader on a particular offset. - sectionReader := io.NewSectionReader(reader, readOffset, missingPartSize) - - // Choose the needed hash algorithms to be calculated by hashCopyBuffer. - // Sha256 is avoided in non-v4 signature requests or HTTPS connections - hashAlgos, hashSums := c.hashMaterials() - - var prtSize int64 - var err error - prtSize, err = hashCopyBuffer(hashAlgos, hashSums, tmpBuffer, sectionReader, readAtBuffer) - if err != nil { - // Send the error back through the channel. - uploadedPartsCh <- uploadedPartRes{ - Size: 0, - Error: err, - } - // Exit the goroutine. - return - } - - // Proceed to upload the part. - var objPart ObjectPart - objPart, err = c.uploadPart(bucketName, objectName, uploadID, tmpBuffer, - uploadReq.PartNum, hashSums["md5"], hashSums["sha256"], prtSize) - if err != nil { - uploadedPartsCh <- uploadedPartRes{ - Size: 0, - Error: err, - } - // Exit the goroutine. - return - } - - // Save successfully uploaded part metadata. - uploadReq.Part = &objPart - - // Send successful part info through the channel. - uploadedPartsCh <- uploadedPartRes{ - Size: missingPartSize, - PartNum: uploadReq.PartNum, - Part: uploadReq.Part, - Error: nil, - } - } - }() - } - - // Gather the responses as they occur and update any - // progress bar. - for u := 1; u <= totalPartsCount; u++ { - uploadRes := <-uploadedPartsCh - if uploadRes.Error != nil { - return totalUploadedSize, uploadRes.Error - } - // Retrieve each uploaded part and store it to be completed. - // part, ok := partsInfo[uploadRes.PartNum] - part := uploadRes.Part - if part == nil { - return 0, ErrInvalidArgument(fmt.Sprintf("Missing part number %d", uploadRes.PartNum)) - } - // Update the totalUploadedSize. - totalUploadedSize += uploadRes.Size - // Update the progress bar if there is one. - if progress != nil { - if _, err = io.CopyN(ioutil.Discard, progress, uploadRes.Size); err != nil { - return totalUploadedSize, err - } - } - // Store the parts to be completed in order. - complMultipartUpload.Parts = append(complMultipartUpload.Parts, CompletePart{ - ETag: part.ETag, - PartNumber: part.PartNumber, - }) - } - - // Verify if we uploaded all the data. - if totalUploadedSize != size { - return totalUploadedSize, ErrUnexpectedEOF(totalUploadedSize, size, bucketName, objectName) - } - - // Sort all completed parts. - sort.Sort(completedParts(complMultipartUpload.Parts)) - _, err = c.completeMultipartUpload(bucketName, objectName, uploadID, complMultipartUpload) - if err != nil { - return totalUploadedSize, err - } - - // Return final size. - return totalUploadedSize, nil -} diff --git a/vendor/src/github.com/minio/minio-go/api-put-object-streaming.go b/vendor/src/github.com/minio/minio-go/api-put-object-streaming.go new file mode 100644 index 000000000..0d4639e83 --- /dev/null +++ b/vendor/src/github.com/minio/minio-go/api-put-object-streaming.go @@ -0,0 +1,436 @@ +/* + * Minio Go Library for Amazon S3 Compatible Cloud Storage (C) 2017 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package minio + +import ( + "fmt" + "io" + "net/http" + "sort" + "strings" + + "github.com/minio/minio-go/pkg/s3utils" +) + +// PutObjectStreaming using AWS streaming signature V4 +func (c Client) PutObjectStreaming(bucketName, objectName string, reader io.Reader) (n int64, err error) { + return c.PutObjectWithProgress(bucketName, objectName, reader, nil, nil) +} + +// putObjectMultipartStream - upload a large object using +// multipart upload and streaming signature for signing payload. +// Comprehensive put object operation involving multipart uploads. +// +// Following code handles these types of readers. +// +// - *os.File +// - *minio.Object +// - Any reader which has a method 'ReadAt()' +// +func (c Client) putObjectMultipartStream(bucketName, objectName string, + reader io.Reader, size int64, metadata map[string][]string, progress io.Reader) (n int64, err error) { + + // Verify if reader is *minio.Object, *os.File or io.ReaderAt. + // NOTE: Verification of object is kept for a specific purpose + // while it is going to be duck typed similar to io.ReaderAt. + // It is to indicate that *minio.Object implements io.ReaderAt. + // and such a functionality is used in the subsequent code path. + if isFile(reader) || !isObject(reader) && isReadAt(reader) { + n, err = c.putObjectMultipartStreamFromReadAt(bucketName, objectName, reader.(io.ReaderAt), size, metadata, progress) + } else { + n, err = c.putObjectMultipartStreamNoChecksum(bucketName, objectName, reader, size, metadata, progress) + } + if err != nil { + errResp := ToErrorResponse(err) + // Verify if multipart functionality is not available, if not + // fall back to single PutObject operation. + if errResp.Code == "AccessDenied" && strings.Contains(errResp.Message, "Access Denied") { + // Verify if size of reader is greater than '5GiB'. + if size > maxSinglePutObjectSize { + return 0, ErrEntityTooLarge(size, maxSinglePutObjectSize, bucketName, objectName) + } + // Fall back to uploading as single PutObject operation. + return c.putObjectNoChecksum(bucketName, objectName, reader, size, metadata, progress) + } + } + return n, err +} + +// uploadedPartRes - the response received from a part upload. +type uploadedPartRes struct { + Error error // Any error encountered while uploading the part. + PartNum int // Number of the part uploaded. + Size int64 // Size of the part uploaded. + Part *ObjectPart +} + +type uploadPartReq struct { + PartNum int // Number of the part uploaded. + Part *ObjectPart // Size of the part uploaded. +} + +// putObjectMultipartFromReadAt - Uploads files bigger than 64MiB. +// Supports all readers which implements io.ReaderAt interface +// (ReadAt method). +// +// NOTE: This function is meant to be used for all readers which +// implement io.ReaderAt which allows us for resuming multipart +// uploads but reading at an offset, which would avoid re-read the +// data which was already uploaded. Internally this function uses +// temporary files for staging all the data, these temporary files are +// cleaned automatically when the caller i.e http client closes the +// stream after uploading all the contents successfully. +func (c Client) putObjectMultipartStreamFromReadAt(bucketName, objectName string, + reader io.ReaderAt, size int64, metadata map[string][]string, progress io.Reader) (n int64, err error) { + // Input validation. + if err = s3utils.CheckValidBucketName(bucketName); err != nil { + return 0, err + } + if err = s3utils.CheckValidObjectName(objectName); err != nil { + return 0, err + } + + // Calculate the optimal parts info for a given size. + totalPartsCount, partSize, lastPartSize, err := optimalPartInfo(size) + if err != nil { + return 0, err + } + + // Initiate a new multipart upload. + uploadID, err := c.newUploadID(bucketName, objectName, metadata) + if err != nil { + return 0, err + } + + // Aborts the multipart upload in progress, if the + // function returns any error, since we do not resume + // we should purge the parts which have been uploaded + // to relinquish storage space. + defer func() { + if err != nil { + c.abortMultipartUpload(bucketName, objectName, uploadID) + } + }() + + // Total data read and written to server. should be equal to 'size' at the end of the call. + var totalUploadedSize int64 + + // Complete multipart upload. + var complMultipartUpload completeMultipartUpload + + // Declare a channel that sends the next part number to be uploaded. + // Buffered to 10000 because thats the maximum number of parts allowed + // by S3. + uploadPartsCh := make(chan uploadPartReq, 10000) + + // Declare a channel that sends back the response of a part upload. + // Buffered to 10000 because thats the maximum number of parts allowed + // by S3. + uploadedPartsCh := make(chan uploadedPartRes, 10000) + + // Used for readability, lastPartNumber is always totalPartsCount. + lastPartNumber := totalPartsCount + + // Send each part number to the channel to be processed. + for p := 1; p <= totalPartsCount; p++ { + uploadPartsCh <- uploadPartReq{PartNum: p, Part: nil} + } + close(uploadPartsCh) + + // Receive each part number from the channel allowing three parallel uploads. + for w := 1; w <= totalWorkers; w++ { + go func() { + // Each worker will draw from the part channel and upload in parallel. + for uploadReq := range uploadPartsCh { + + // If partNumber was not uploaded we calculate the missing + // part offset and size. For all other part numbers we + // calculate offset based on multiples of partSize. + readOffset := int64(uploadReq.PartNum-1) * partSize + + // As a special case if partNumber is lastPartNumber, we + // calculate the offset based on the last part size. + if uploadReq.PartNum == lastPartNumber { + readOffset = (size - lastPartSize) + partSize = lastPartSize + } + + // Get a section reader on a particular offset. + sectionReader := newHook(io.NewSectionReader(reader, readOffset, partSize), progress) + + // Proceed to upload the part. + var objPart ObjectPart + objPart, err = c.uploadPart(bucketName, objectName, uploadID, + sectionReader, uploadReq.PartNum, + nil, nil, partSize, metadata) + if err != nil { + uploadedPartsCh <- uploadedPartRes{ + Size: 0, + Error: err, + } + // Exit the goroutine. + return + } + + // Save successfully uploaded part metadata. + uploadReq.Part = &objPart + + // Send successful part info through the channel. + uploadedPartsCh <- uploadedPartRes{ + Size: objPart.Size, + PartNum: uploadReq.PartNum, + Part: uploadReq.Part, + Error: nil, + } + } + }() + } + + // Gather the responses as they occur and update any + // progress bar. + for u := 1; u <= totalPartsCount; u++ { + uploadRes := <-uploadedPartsCh + if uploadRes.Error != nil { + return totalUploadedSize, uploadRes.Error + } + // Retrieve each uploaded part and store it to be completed. + // part, ok := partsInfo[uploadRes.PartNum] + part := uploadRes.Part + if part == nil { + return 0, ErrInvalidArgument(fmt.Sprintf("Missing part number %d", uploadRes.PartNum)) + } + // Update the totalUploadedSize. + totalUploadedSize += uploadRes.Size + // Store the parts to be completed in order. + complMultipartUpload.Parts = append(complMultipartUpload.Parts, CompletePart{ + ETag: part.ETag, + PartNumber: part.PartNumber, + }) + } + + // Verify if we uploaded all the data. + if totalUploadedSize != size { + return totalUploadedSize, ErrUnexpectedEOF(totalUploadedSize, size, bucketName, objectName) + } + + // Sort all completed parts. + sort.Sort(completedParts(complMultipartUpload.Parts)) + _, err = c.completeMultipartUpload(bucketName, objectName, uploadID, complMultipartUpload) + if err != nil { + return totalUploadedSize, err + } + + // Return final size. + return totalUploadedSize, nil +} + +func (c Client) putObjectMultipartStreamNoChecksum(bucketName, objectName string, + reader io.Reader, size int64, metadata map[string][]string, progress io.Reader) (n int64, err error) { + // Input validation. + if err = s3utils.CheckValidBucketName(bucketName); err != nil { + return 0, err + } + if err = s3utils.CheckValidObjectName(objectName); err != nil { + return 0, err + } + + // Calculate the optimal parts info for a given size. + totalPartsCount, partSize, lastPartSize, err := optimalPartInfo(size) + if err != nil { + return 0, err + } + + // Initiates a new multipart request + uploadID, err := c.newUploadID(bucketName, objectName, metadata) + if err != nil { + return 0, err + } + + // Aborts the multipart upload if the function returns + // any error, since we do not resume we should purge + // the parts which have been uploaded to relinquish + // storage space. + defer func() { + if err != nil { + c.abortMultipartUpload(bucketName, objectName, uploadID) + } + }() + + // Total data read and written to server. should be equal to 'size' at the end of the call. + var totalUploadedSize int64 + + // Initialize parts uploaded map. + partsInfo := make(map[int]ObjectPart) + + // Part number always starts with '1'. + var partNumber int + for partNumber = 1; partNumber <= totalPartsCount; partNumber++ { + // Update progress reader appropriately to the latest offset + // as we read from the source. + hookReader := newHook(reader, progress) + + // Proceed to upload the part. + if partNumber == totalPartsCount { + partSize = lastPartSize + } + + var objPart ObjectPart + objPart, err = c.uploadPart(bucketName, objectName, uploadID, + io.LimitReader(hookReader, partSize), + partNumber, nil, nil, partSize, metadata) + if err != nil { + return totalUploadedSize, err + } + + // Save successfully uploaded part metadata. + partsInfo[partNumber] = objPart + + // Save successfully uploaded size. + totalUploadedSize += partSize + } + + // Verify if we uploaded all the data. + if size > 0 { + if totalUploadedSize != size { + return totalUploadedSize, ErrUnexpectedEOF(totalUploadedSize, size, bucketName, objectName) + } + } + + // Complete multipart upload. + var complMultipartUpload completeMultipartUpload + + // Loop over total uploaded parts to save them in + // Parts array before completing the multipart request. + for i := 1; i < partNumber; i++ { + part, ok := partsInfo[i] + if !ok { + return 0, ErrInvalidArgument(fmt.Sprintf("Missing part number %d", i)) + } + complMultipartUpload.Parts = append(complMultipartUpload.Parts, CompletePart{ + ETag: part.ETag, + PartNumber: part.PartNumber, + }) + } + + // Sort all completed parts. + sort.Sort(completedParts(complMultipartUpload.Parts)) + _, err = c.completeMultipartUpload(bucketName, objectName, uploadID, complMultipartUpload) + if err != nil { + return totalUploadedSize, err + } + + // Return final size. + return totalUploadedSize, nil +} + +// putObjectNoChecksum special function used Google Cloud Storage. This special function +// is used for Google Cloud Storage since Google's multipart API is not S3 compatible. +func (c Client) putObjectNoChecksum(bucketName, objectName string, reader io.Reader, size int64, metaData map[string][]string, progress io.Reader) (n int64, err error) { + // Input validation. + if err := s3utils.CheckValidBucketName(bucketName); err != nil { + return 0, err + } + if err := s3utils.CheckValidObjectName(objectName); err != nil { + return 0, err + } + + // Size -1 is only supported on Google Cloud Storage, we error + // out in all other situations. + if size < 0 && !s3utils.IsGoogleEndpoint(c.endpointURL) { + return 0, ErrEntityTooSmall(size, bucketName, objectName) + } + if size > 0 { + if isReadAt(reader) && !isObject(reader) { + reader = io.NewSectionReader(reader.(io.ReaderAt), 0, size) + } + } + + // Update progress reader appropriately to the latest offset as we + // read from the source. + readSeeker := newHook(reader, progress) + + // This function does not calculate sha256 and md5sum for payload. + // Execute put object. + st, err := c.putObjectDo(bucketName, objectName, readSeeker, nil, nil, size, metaData) + if err != nil { + return 0, err + } + if st.Size != size { + return 0, ErrUnexpectedEOF(st.Size, size, bucketName, objectName) + } + return size, nil +} + +// putObjectDo - executes the put object http operation. +// NOTE: You must have WRITE permissions on a bucket to add an object to it. +func (c Client) putObjectDo(bucketName, objectName string, reader io.Reader, md5Sum []byte, sha256Sum []byte, size int64, metaData map[string][]string) (ObjectInfo, error) { + // Input validation. + if err := s3utils.CheckValidBucketName(bucketName); err != nil { + return ObjectInfo{}, err + } + if err := s3utils.CheckValidObjectName(objectName); err != nil { + return ObjectInfo{}, err + } + + // Set headers. + customHeader := make(http.Header) + + // Set metadata to headers + for k, v := range metaData { + if len(v) > 0 { + customHeader.Set(k, v[0]) + } + } + + // If Content-Type is not provided, set the default application/octet-stream one + if v, ok := metaData["Content-Type"]; !ok || len(v) == 0 { + customHeader.Set("Content-Type", "application/octet-stream") + } + + // Populate request metadata. + reqMetadata := requestMetadata{ + bucketName: bucketName, + objectName: objectName, + customHeader: customHeader, + contentBody: reader, + contentLength: size, + contentMD5Bytes: md5Sum, + contentSHA256Bytes: sha256Sum, + } + + // Execute PUT an objectName. + resp, err := c.executeMethod("PUT", reqMetadata) + defer closeResponse(resp) + if err != nil { + return ObjectInfo{}, err + } + if resp != nil { + if resp.StatusCode != http.StatusOK { + return ObjectInfo{}, httpRespToErrorResponse(resp, bucketName, objectName) + } + } + + var objInfo ObjectInfo + // Trim off the odd double quotes from ETag in the beginning and end. + objInfo.ETag = strings.TrimPrefix(resp.Header.Get("ETag"), "\"") + objInfo.ETag = strings.TrimSuffix(objInfo.ETag, "\"") + // A success here means data was written to server successfully. + objInfo.Size = size + + // Return here. + return objInfo, nil +} diff --git a/vendor/src/github.com/minio/minio-go/api-put-object.go b/vendor/src/github.com/minio/minio-go/api-put-object.go index fbcfb171e..1fa0a9238 100644 --- a/vendor/src/github.com/minio/minio-go/api-put-object.go +++ b/vendor/src/github.com/minio/minio-go/api-put-object.go @@ -18,13 +18,12 @@ package minio import ( "io" - "io/ioutil" - "net/http" "os" "reflect" "runtime" "strings" + "github.com/minio/minio-go/pkg/credentials" "github.com/minio/minio-go/pkg/s3utils" ) @@ -143,164 +142,79 @@ func (a completedParts) Less(i, j int) bool { return a[i].PartNumber < a[j].Part // // You must have WRITE permissions on a bucket to create an object. // -// - For size smaller than 64MiB PutObject automatically does a single atomic Put operation. -// - For size larger than 64MiB PutObject automatically does a multipart Put operation. -// - For size input as -1 PutObject does a multipart Put operation until input stream reaches EOF. -// Maximum object size that can be uploaded through this operation will be 5TiB. -// -// NOTE: Google Cloud Storage does not implement Amazon S3 Compatible multipart PUT. -// So we fall back to single PUT operation with the maximum limit of 5GiB. +// - For size smaller than 64MiB PutObject automatically does a +// single atomic Put operation. +// - For size larger than 64MiB PutObject automatically does a +// multipart Put operation. +// - For size input as -1 PutObject does a multipart Put operation +// until input stream reaches EOF. Maximum object size that can +// be uploaded through this operation will be 5TiB. func (c Client) PutObject(bucketName, objectName string, reader io.Reader, contentType string) (n int64, err error) { - return c.PutObjectWithProgress(bucketName, objectName, reader, contentType, nil) + return c.PutObjectWithMetadata(bucketName, objectName, reader, map[string][]string{ + "Content-Type": []string{contentType}, + }, nil) } -// putObjectNoChecksum special function used Google Cloud Storage. This special function -// is used for Google Cloud Storage since Google's multipart API is not S3 compatible. -func (c Client) putObjectNoChecksum(bucketName, objectName string, reader io.Reader, size int64, metaData map[string][]string, progress io.Reader) (n int64, err error) { - // Input validation. - if err := s3utils.CheckValidBucketName(bucketName); err != nil { - return 0, err - } - if err := s3utils.CheckValidObjectName(objectName); err != nil { - return 0, err - } - if size > 0 { - readerAt, ok := reader.(io.ReaderAt) - if ok { - reader = io.NewSectionReader(readerAt, 0, size) - } - } +// PutObjectWithSize - is a helper PutObject similar in behavior to PutObject() +// but takes the size argument explicitly, this function avoids doing reflection +// internally to figure out the size of input stream. Also if the input size is +// lesser than 0 this function returns an error. +func (c Client) PutObjectWithSize(bucketName, objectName string, reader io.Reader, readerSize int64, metadata map[string][]string, progress io.Reader) (n int64, err error) { + return c.putObjectCommon(bucketName, objectName, reader, readerSize, metadata, progress) +} - // Update progress reader appropriately to the latest offset as we - // read from the source. - readSeeker := newHook(reader, progress) +// PutObjectWithMetadata using AWS streaming signature V4 +func (c Client) PutObjectWithMetadata(bucketName, objectName string, reader io.Reader, metadata map[string][]string, progress io.Reader) (n int64, err error) { + return c.PutObjectWithProgress(bucketName, objectName, reader, metadata, progress) +} - // This function does not calculate sha256 and md5sum for payload. - // Execute put object. - st, err := c.putObjectDo(bucketName, objectName, readSeeker, nil, nil, size, metaData) +// PutObjectWithProgress using AWS streaming signature V4 +func (c Client) PutObjectWithProgress(bucketName, objectName string, reader io.Reader, metadata map[string][]string, progress io.Reader) (n int64, err error) { + // Size of the object. + var size int64 + + // Get reader size. + size, err = getReaderSize(reader) if err != nil { return 0, err } - if st.Size != size { - return 0, ErrUnexpectedEOF(st.Size, size, bucketName, objectName) - } - return size, nil + return c.putObjectCommon(bucketName, objectName, reader, size, metadata, progress) } -// putObjectSingle is a special function for uploading single put object request. -// This special function is used as a fallback when multipart upload fails. -func (c Client) putObjectSingle(bucketName, objectName string, reader io.Reader, size int64, metaData map[string][]string, progress io.Reader) (n int64, err error) { - // Input validation. - if err := s3utils.CheckValidBucketName(bucketName); err != nil { - return 0, err - } - if err := s3utils.CheckValidObjectName(objectName); err != nil { - return 0, err - } - if size > maxSinglePutObjectSize { - return 0, ErrEntityTooLarge(size, maxSinglePutObjectSize, bucketName, objectName) - } - // If size is a stream, upload up to 5GiB. - if size <= -1 { - size = maxSinglePutObjectSize +func (c Client) putObjectCommon(bucketName, objectName string, reader io.Reader, size int64, metadata map[string][]string, progress io.Reader) (n int64, err error) { + // Check for largest object size allowed. + if size > int64(maxMultipartPutObjectSize) { + return 0, ErrEntityTooLarge(size, maxMultipartPutObjectSize, bucketName, objectName) } - // Add the appropriate hash algorithms that need to be calculated by hashCopyN - // In case of non-v4 signature request or HTTPS connection, sha256 is not needed. - hashAlgos, hashSums := c.hashMaterials() - - // Initialize a new temporary file. - tmpFile, err := newTempFile("single$-putobject-single") - if err != nil { - return 0, err - } - defer tmpFile.Close() - - size, err = hashCopyN(hashAlgos, hashSums, tmpFile, reader, size) - // Return error if its not io.EOF. - if err != nil && err != io.EOF { - return 0, err + // NOTE: Streaming signature is not supported by GCS. + if s3utils.IsGoogleEndpoint(c.endpointURL) { + // Do not compute MD5 for Google Cloud Storage. + return c.putObjectNoChecksum(bucketName, objectName, reader, size, metadata, progress) } - // Seek back to beginning of the temporary file. - if _, err = tmpFile.Seek(0, 0); err != nil { - return 0, err - } - reader = tmpFile - - // Execute put object. - st, err := c.putObjectDo(bucketName, objectName, reader, hashSums["md5"], hashSums["sha256"], size, metaData) - if err != nil { - return 0, err - } - if st.Size != size { - return 0, ErrUnexpectedEOF(st.Size, size, bucketName, objectName) - } - // Progress the reader to the size if putObjectDo is successful. - if progress != nil { - if _, err = io.CopyN(ioutil.Discard, progress, size); err != nil { - return size, err + if c.overrideSignerType.IsV2() { + if size >= 0 && size < minPartSize { + return c.putObjectNoChecksum(bucketName, objectName, reader, size, metadata, progress) } + return c.putObjectMultipart(bucketName, objectName, reader, size, metadata, progress) } - return size, nil -} - -// putObjectDo - executes the put object http operation. -// NOTE: You must have WRITE permissions on a bucket to add an object to it. -func (c Client) putObjectDo(bucketName, objectName string, reader io.Reader, md5Sum []byte, sha256Sum []byte, size int64, metaData map[string][]string) (ObjectInfo, error) { - // Input validation. - if err := s3utils.CheckValidBucketName(bucketName); err != nil { - return ObjectInfo{}, err - } - if err := s3utils.CheckValidObjectName(objectName); err != nil { - return ObjectInfo{}, err - } - - // Set headers. - customHeader := make(http.Header) - - // Set metadata to headers - for k, v := range metaData { - if len(v) > 0 { - customHeader.Set(k, v[0]) - } - } - - // If Content-Type is not provided, set the default application/octet-stream one - if v, ok := metaData["Content-Type"]; !ok || len(v) == 0 { - customHeader.Set("Content-Type", "application/octet-stream") - } - - // Populate request metadata. - reqMetadata := requestMetadata{ - bucketName: bucketName, - objectName: objectName, - customHeader: customHeader, - contentBody: reader, - contentLength: size, - contentMD5Bytes: md5Sum, - contentSHA256Bytes: sha256Sum, - } - - // Execute PUT an objectName. - resp, err := c.executeMethod("PUT", reqMetadata) - defer closeResponse(resp) - if err != nil { - return ObjectInfo{}, err - } - if resp != nil { - if resp.StatusCode != http.StatusOK { - return ObjectInfo{}, httpRespToErrorResponse(resp, bucketName, objectName) - } - } - - var objInfo ObjectInfo - // Trim off the odd double quotes from ETag in the beginning and end. - objInfo.ETag = strings.TrimPrefix(resp.Header.Get("ETag"), "\"") - objInfo.ETag = strings.TrimSuffix(objInfo.ETag, "\"") - // A success here means data was written to server successfully. - objInfo.Size = size - - // Return here. - return objInfo, nil + + // If size cannot be found on a stream, it is not possible + // to upload using streaming signature, fall back to multipart. + if size < 0 { + // Set regular signature calculation. + c.overrideSignerType = credentials.SignatureV4 + return c.putObjectMultipart(bucketName, objectName, reader, size, metadata, progress) + } + + // Set streaming signature. + c.overrideSignerType = credentials.SignatureV4Streaming + + if size < minPartSize { + return c.putObjectNoChecksum(bucketName, objectName, reader, size, metadata, progress) + } + + // For all sizes greater than 64MiB do multipart. + return c.putObjectMultipartStream(bucketName, objectName, reader, size, metadata, progress) } diff --git a/vendor/src/github.com/minio/minio-go/api-stat.go b/vendor/src/github.com/minio/minio-go/api-stat.go index 4b530327b..5f06bfc9e 100644 --- a/vendor/src/github.com/minio/minio-go/api-stat.go +++ b/vendor/src/github.com/minio/minio-go/api-stat.go @@ -167,11 +167,6 @@ func (c Client) statObject(bucketName, objectName string, reqHeaders RequestHead contentType = "application/octet-stream" } - // Extract only the relevant header keys describing the object. - // following function filters out a list of standard set of keys - // which are not part of object metadata. - metadata := extractObjMetadata(resp.Header) - // Save object metadata info. return ObjectInfo{ ETag: md5sum, @@ -179,6 +174,9 @@ func (c Client) statObject(bucketName, objectName string, reqHeaders RequestHead Size: size, LastModified: date, ContentType: contentType, - Metadata: metadata, + // Extract only the relevant header keys describing the object. + // following function filters out a list of standard set of keys + // which are not part of object metadata. + Metadata: extractObjMetadata(resp.Header), }, nil } diff --git a/vendor/src/github.com/minio/minio-go/api.go b/vendor/src/github.com/minio/minio-go/api.go index 3020e5b9d..a25acc338 100644 --- a/vendor/src/github.com/minio/minio-go/api.go +++ b/vendor/src/github.com/minio/minio-go/api.go @@ -87,7 +87,7 @@ type Client struct { // Global constants. const ( libraryName = "minio-go" - libraryVersion = "2.1.0" + libraryVersion = "3.0.0" ) // User Agent should always following the below style. @@ -211,7 +211,7 @@ func privateNew(endpoint string, creds *credentials.Credentials, secure bool, re // Instantiate http client and bucket location cache. clnt.httpClient = &http.Client{ - Transport: http.DefaultTransport, + Transport: defaultMinioTransport, CheckRedirect: redirectHeaders, } diff --git a/vendor/src/github.com/minio/minio-go/api_functional_v2_test.go b/vendor/src/github.com/minio/minio-go/api_functional_v2_test.go deleted file mode 100644 index 2b0ec43a0..000000000 --- a/vendor/src/github.com/minio/minio-go/api_functional_v2_test.go +++ /dev/null @@ -1,1108 +0,0 @@ -/* - * Minio Go Library for Amazon S3 Compatible Cloud Storage (C) 2015 Minio, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package minio - -import ( - "bytes" - "errors" - "io" - "io/ioutil" - "math/rand" - "net/http" - "net/url" - "os" - "testing" - "time" - - "github.com/minio/minio-go/pkg/policy" -) - -// Tests bucket re-create errors. -func TestMakeBucketErrorV2(t *testing.T) { - if testing.Short() { - t.Skip("skipping functional tests for short runs") - } - if os.Getenv(serverEndpoint) != "s3.amazonaws.com" { - t.Skip("skipping region functional tests for non s3 runs") - } - - // Seed random based on current time. - rand.Seed(time.Now().Unix()) - - // Instantiate new minio client object. - c, err := NewV2( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableSecurity)), - ) - if err != nil { - t.Fatal("Error:", err) - } - - // Enable tracing, write to stderr. - // c.TraceOn(os.Stderr) - - // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") - - // Generate a new random bucket name. - bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") - - // Make a new bucket in 'eu-west-1'. - if err = c.MakeBucket(bucketName, "eu-west-1"); err != nil { - t.Fatal("Error:", err, bucketName) - } - if err = c.MakeBucket(bucketName, "eu-west-1"); err == nil { - t.Fatal("Error: make bucket should should fail for", bucketName) - } - // Verify valid error response from server. - if ToErrorResponse(err).Code != "BucketAlreadyExists" && - ToErrorResponse(err).Code != "BucketAlreadyOwnedByYou" { - t.Fatal("Error: Invalid error returned by server", err) - } - if err = c.RemoveBucket(bucketName); err != nil { - t.Fatal("Error:", err, bucketName) - } -} - -// Test get object reader to not throw error on being closed twice. -func TestGetObjectClosedTwiceV2(t *testing.T) { - if testing.Short() { - t.Skip("skipping functional tests for short runs") - } - - // Seed random based on current time. - rand.Seed(time.Now().Unix()) - - // Instantiate new minio client object. - c, err := NewV2( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableSecurity)), - ) - if err != nil { - t.Fatal("Error:", err) - } - - // Enable tracing, write to stderr. - // c.TraceOn(os.Stderr) - - // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") - - // Generate a new random bucket name. - bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") - - // Make a new bucket. - err = c.MakeBucket(bucketName, "us-east-1") - if err != nil { - t.Fatal("Error:", err, bucketName) - } - - // Generate data more than 32K. - buf := bytes.Repeat([]byte("h"), rand.Intn(1<<20)+32*1024) - - // Save the data - objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") - n, err := c.PutObject(bucketName, objectName, bytes.NewReader(buf), "binary/octet-stream") - if err != nil { - t.Fatal("Error:", err, bucketName, objectName) - } - - if n != int64(len(buf)) { - t.Fatalf("Error: number of bytes does not match, want %v, got %v\n", len(buf), n) - } - - // Read the data back - r, err := c.GetObject(bucketName, objectName) - if err != nil { - t.Fatal("Error:", err, bucketName, objectName) - } - - st, err := r.Stat() - if err != nil { - t.Fatal("Error:", err, bucketName, objectName) - } - if st.Size != int64(len(buf)) { - t.Fatalf("Error: number of bytes in stat does not match, want %v, got %v\n", - len(buf), st.Size) - } - if err := r.Close(); err != nil { - t.Fatal("Error:", err) - } - if err := r.Close(); err == nil { - t.Fatal("Error: object is already closed, should return error") - } - - err = c.RemoveObject(bucketName, objectName) - if err != nil { - t.Fatal("Error: ", err) - } - err = c.RemoveBucket(bucketName) - if err != nil { - t.Fatal("Error:", err) - } -} - -// Tests removing partially uploaded objects. -func TestRemovePartiallyUploadedV2(t *testing.T) { - if testing.Short() { - t.Skip("skipping function tests for short runs") - } - - // Seed random based on current time. - rand.Seed(time.Now().Unix()) - - // Instantiate new minio client object. - c, err := NewV2( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableSecurity)), - ) - if err != nil { - t.Fatal("Error:", err) - } - - // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") - - // Enable tracing, write to stdout. - // c.TraceOn(os.Stderr) - - // Generate a new random bucket name. - bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") - - // make a new bucket. - err = c.MakeBucket(bucketName, "us-east-1") - if err != nil { - t.Fatal("Error:", err, bucketName) - } - - r := bytes.NewReader(bytes.Repeat([]byte("a"), 128*1024)) - - reader, writer := io.Pipe() - go func() { - i := 0 - for i < 25 { - _, cerr := io.CopyN(writer, r, 128*1024) - if cerr != nil { - t.Fatal("Error:", cerr, bucketName) - } - i++ - r.Seek(0, 0) - } - writer.CloseWithError(errors.New("proactively closed to be verified later")) - }() - - objectName := bucketName + "-resumable" - _, err = c.PutObject(bucketName, objectName, reader, "application/octet-stream") - if err == nil { - t.Fatal("Error: PutObject should fail.") - } - if err.Error() != "proactively closed to be verified later" { - t.Fatal("Error:", err) - } - err = c.RemoveIncompleteUpload(bucketName, objectName) - if err != nil { - t.Fatal("Error:", err) - } - err = c.RemoveBucket(bucketName) - if err != nil { - t.Fatal("Error:", err) - } -} - -// Tests FPutObject hidden contentType setting -func TestFPutObjectV2(t *testing.T) { - if testing.Short() { - t.Skip("skipping functional tests for short runs") - } - - // Seed random based on current time. - rand.Seed(time.Now().Unix()) - - // Instantiate new minio client object. - c, err := NewV2( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableSecurity)), - ) - if err != nil { - t.Fatal("Error:", err) - } - - // Enable tracing, write to stderr. - // c.TraceOn(os.Stderr) - - // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") - - // Generate a new random bucket name. - bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") - - // Make a new bucket. - err = c.MakeBucket(bucketName, "us-east-1") - if err != nil { - t.Fatal("Error:", err, bucketName) - } - - // Make a temp file with 11*1024*1024 bytes of data. - file, err := ioutil.TempFile(os.TempDir(), "FPutObjectTest") - if err != nil { - t.Fatal("Error:", err) - } - - r := bytes.NewReader(bytes.Repeat([]byte("b"), 11*1024*1024)) - n, err := io.CopyN(file, r, 11*1024*1024) - if err != nil { - t.Fatal("Error:", err) - } - if n != int64(11*1024*1024) { - t.Fatalf("Error: number of bytes does not match, want %v, got %v\n", 11*1024*1024, n) - } - - // Close the file pro-actively for windows. - err = file.Close() - if err != nil { - t.Fatal("Error:", err) - } - - // Set base object name - objectName := bucketName + "FPutObject" - - // Perform standard FPutObject with contentType provided (Expecting application/octet-stream) - n, err = c.FPutObject(bucketName, objectName+"-standard", file.Name(), "application/octet-stream") - if err != nil { - t.Fatal("Error:", err) - } - if n != int64(11*1024*1024) { - t.Fatalf("Error: number of bytes does not match, want %v, got %v\n", 11*1024*1024, n) - } - - // Perform FPutObject with no contentType provided (Expecting application/octet-stream) - n, err = c.FPutObject(bucketName, objectName+"-Octet", file.Name(), "") - if err != nil { - t.Fatal("Error:", err) - } - if n != int64(11*1024*1024) { - t.Fatalf("Error: number of bytes does not match, want %v, got %v\n", 11*1024*1024, n) - } - - // Add extension to temp file name - fileName := file.Name() - err = os.Rename(file.Name(), fileName+".gtar") - if err != nil { - t.Fatal("Error:", err) - } - - // Perform FPutObject with no contentType provided (Expecting application/x-gtar) - n, err = c.FPutObject(bucketName, objectName+"-GTar", fileName+".gtar", "") - if err != nil { - t.Fatal("Error:", err) - } - if n != int64(11*1024*1024) { - t.Fatalf("Error: number of bytes does not match, want %v, got %v\n", 11*1024*1024, n) - } - - // Check headers - rStandard, err := c.StatObject(bucketName, objectName+"-standard") - if err != nil { - t.Fatal("Error:", err, bucketName, objectName+"-standard") - } - if rStandard.ContentType != "application/octet-stream" { - t.Fatalf("Error: Content-Type headers mismatched, want %v, got %v\n", - "application/octet-stream", rStandard.ContentType) - } - - rOctet, err := c.StatObject(bucketName, objectName+"-Octet") - if err != nil { - t.Fatal("Error:", err, bucketName, objectName+"-Octet") - } - if rOctet.ContentType != "application/octet-stream" { - t.Fatalf("Error: Content-Type headers mismatched, want %v, got %v\n", - "application/octet-stream", rStandard.ContentType) - } - - rGTar, err := c.StatObject(bucketName, objectName+"-GTar") - if err != nil { - t.Fatal("Error:", err, bucketName, objectName+"-GTar") - } - if rGTar.ContentType != "application/x-gtar" { - t.Fatalf("Error: Content-Type headers mismatched, want %v, got %v\n", - "application/x-gtar", rStandard.ContentType) - } - - // Remove all objects and bucket and temp file - err = c.RemoveObject(bucketName, objectName+"-standard") - if err != nil { - t.Fatal("Error: ", err) - } - - err = c.RemoveObject(bucketName, objectName+"-Octet") - if err != nil { - t.Fatal("Error: ", err) - } - - err = c.RemoveObject(bucketName, objectName+"-GTar") - if err != nil { - t.Fatal("Error: ", err) - } - - err = c.RemoveBucket(bucketName) - if err != nil { - t.Fatal("Error:", err) - } - - err = os.Remove(fileName + ".gtar") - if err != nil { - t.Fatal("Error:", err) - } - -} - -// Tests various bucket supported formats. -func TestMakeBucketRegionsV2(t *testing.T) { - if testing.Short() { - t.Skip("skipping functional tests for short runs") - } - if os.Getenv(serverEndpoint) != "s3.amazonaws.com" { - t.Skip("skipping region functional tests for non s3 runs") - } - - // Seed random based on current time. - rand.Seed(time.Now().Unix()) - - // Instantiate new minio client object. - c, err := NewV2( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableSecurity)), - ) - if err != nil { - t.Fatal("Error:", err) - } - - // Enable tracing, write to stderr. - // c.TraceOn(os.Stderr) - - // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") - - // Generate a new random bucket name. - bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") - - // Make a new bucket in 'eu-central-1'. - if err = c.MakeBucket(bucketName, "eu-west-1"); err != nil { - t.Fatal("Error:", err, bucketName) - } - - if err = c.RemoveBucket(bucketName); err != nil { - t.Fatal("Error:", err, bucketName) - } - - // Make a new bucket with '.' in its name, in 'us-west-2'. This - // request is internally staged into a path style instead of - // virtual host style. - if err = c.MakeBucket(bucketName+".withperiod", "us-west-2"); err != nil { - t.Fatal("Error:", err, bucketName+".withperiod") - } - - // Remove the newly created bucket. - if err = c.RemoveBucket(bucketName + ".withperiod"); err != nil { - t.Fatal("Error:", err, bucketName+".withperiod") - } -} - -// Tests get object ReaderSeeker interface methods. -func TestGetObjectReadSeekFunctionalV2(t *testing.T) { - if testing.Short() { - t.Skip("skipping functional tests for short runs") - } - - // Seed random based on current time. - rand.Seed(time.Now().Unix()) - - // Instantiate new minio client object. - c, err := NewV2( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableSecurity)), - ) - if err != nil { - t.Fatal("Error:", err) - } - - // Enable tracing, write to stderr. - // c.TraceOn(os.Stderr) - - // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") - - // Generate a new random bucket name. - bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") - - // Make a new bucket. - err = c.MakeBucket(bucketName, "us-east-1") - if err != nil { - t.Fatal("Error:", err, bucketName) - } - - // Generate data more than 32K. - buf := bytes.Repeat([]byte("2"), rand.Intn(1<<20)+32*1024) - - // Save the data. - objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") - n, err := c.PutObject(bucketName, objectName, bytes.NewReader(buf), "binary/octet-stream") - if err != nil { - t.Fatal("Error:", err, bucketName, objectName) - } - - if n != int64(len(buf)) { - t.Fatalf("Error: number of bytes does not match, want %v, got %v\n", len(buf), n) - } - - // Read the data back - r, err := c.GetObject(bucketName, objectName) - if err != nil { - t.Fatal("Error:", err, bucketName, objectName) - } - - st, err := r.Stat() - if err != nil { - t.Fatal("Error:", err, bucketName, objectName) - } - if st.Size != int64(len(buf)) { - t.Fatalf("Error: number of bytes in stat does not match, want %v, got %v\n", - len(buf), st.Size) - } - - offset := int64(2048) - n, err = r.Seek(offset, 0) - if err != nil { - t.Fatal("Error:", err, offset) - } - if n != offset { - t.Fatalf("Error: number of bytes seeked does not match, want %v, got %v\n", - offset, n) - } - n, err = r.Seek(0, 1) - if err != nil { - t.Fatal("Error:", err) - } - if n != offset { - t.Fatalf("Error: number of current seek does not match, want %v, got %v\n", - offset, n) - } - _, err = r.Seek(offset, 2) - if err == nil { - t.Fatal("Error: seek on positive offset for whence '2' should error out") - } - n, err = r.Seek(-offset, 2) - if err != nil { - t.Fatal("Error:", err) - } - if n != st.Size-offset { - t.Fatalf("Error: number of bytes seeked back does not match, want %d, got %v\n", st.Size-offset, n) - } - - var buffer1 bytes.Buffer - if _, err = io.CopyN(&buffer1, r, st.Size); err != nil { - if err != io.EOF { - t.Fatal("Error:", err) - } - } - if !bytes.Equal(buf[len(buf)-int(offset):], buffer1.Bytes()) { - t.Fatal("Error: Incorrect read bytes v/s original buffer.") - } - - // Seek again and read again. - n, err = r.Seek(offset-1, 0) - if err != nil { - t.Fatal("Error:", err) - } - if n != (offset - 1) { - t.Fatalf("Error: number of bytes seeked back does not match, want %v, got %v\n", offset-1, n) - } - - var buffer2 bytes.Buffer - if _, err = io.CopyN(&buffer2, r, st.Size); err != nil { - if err != io.EOF { - t.Fatal("Error:", err) - } - } - // Verify now lesser bytes. - if !bytes.Equal(buf[2047:], buffer2.Bytes()) { - t.Fatal("Error: Incorrect read bytes v/s original buffer.") - } - - err = c.RemoveObject(bucketName, objectName) - if err != nil { - t.Fatal("Error: ", err) - } - err = c.RemoveBucket(bucketName) - if err != nil { - t.Fatal("Error:", err) - } -} - -// Tests get object ReaderAt interface methods. -func TestGetObjectReadAtFunctionalV2(t *testing.T) { - if testing.Short() { - t.Skip("skipping functional tests for the short runs") - } - - // Seed random based on current time. - rand.Seed(time.Now().Unix()) - - // Instantiate new minio client object. - c, err := NewV2( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableSecurity)), - ) - if err != nil { - t.Fatal("Error:", err) - } - - // Enable tracing, write to stderr. - // c.TraceOn(os.Stderr) - - // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") - - // Generate a new random bucket name. - bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") - - // Make a new bucket. - err = c.MakeBucket(bucketName, "us-east-1") - if err != nil { - t.Fatal("Error:", err, bucketName) - } - - // Generate data more than 32K - buf := bytes.Repeat([]byte("8"), rand.Intn(1<<20)+32*1024) - - // Save the data - objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") - n, err := c.PutObject(bucketName, objectName, bytes.NewReader(buf), "binary/octet-stream") - if err != nil { - t.Fatal("Error:", err, bucketName, objectName) - } - - if n != int64(len(buf)) { - t.Fatalf("Error: number of bytes does not match, want %v, got %v\n", len(buf), n) - } - - // Read the data back - r, err := c.GetObject(bucketName, objectName) - if err != nil { - t.Fatal("Error:", err, bucketName, objectName) - } - - st, err := r.Stat() - if err != nil { - t.Fatal("Error:", err, bucketName, objectName) - } - if st.Size != int64(len(buf)) { - t.Fatalf("Error: number of bytes in stat does not match, want %v, got %v\n", - len(buf), st.Size) - } - - offset := int64(2048) - - // Read directly - buf2 := make([]byte, 512) - buf3 := make([]byte, 512) - buf4 := make([]byte, 512) - - m, err := r.ReadAt(buf2, offset) - if err != nil { - t.Fatal("Error:", err, st.Size, len(buf2), offset) - } - if m != len(buf2) { - t.Fatalf("Error: ReadAt read shorter bytes before reaching EOF, want %v, got %v\n", m, len(buf2)) - } - if !bytes.Equal(buf2, buf[offset:offset+512]) { - t.Fatal("Error: Incorrect read between two ReadAt from same offset.") - } - offset += 512 - m, err = r.ReadAt(buf3, offset) - if err != nil { - t.Fatal("Error:", err, st.Size, len(buf3), offset) - } - if m != len(buf3) { - t.Fatalf("Error: ReadAt read shorter bytes before reaching EOF, want %v, got %v\n", m, len(buf3)) - } - if !bytes.Equal(buf3, buf[offset:offset+512]) { - t.Fatal("Error: Incorrect read between two ReadAt from same offset.") - } - offset += 512 - m, err = r.ReadAt(buf4, offset) - if err != nil { - t.Fatal("Error:", err, st.Size, len(buf4), offset) - } - if m != len(buf4) { - t.Fatalf("Error: ReadAt read shorter bytes before reaching EOF, want %v, got %v\n", m, len(buf4)) - } - if !bytes.Equal(buf4, buf[offset:offset+512]) { - t.Fatal("Error: Incorrect read between two ReadAt from same offset.") - } - - buf5 := make([]byte, n) - // Read the whole object. - m, err = r.ReadAt(buf5, 0) - if err != nil { - if err != io.EOF { - t.Fatal("Error:", err, len(buf5)) - } - } - if m != len(buf5) { - t.Fatalf("Error: ReadAt read shorter bytes before reaching EOF, want %v, got %v\n", m, len(buf5)) - } - if !bytes.Equal(buf, buf5) { - t.Fatal("Error: Incorrect data read in GetObject, than what was previously upoaded.") - } - - buf6 := make([]byte, n+1) - // Read the whole object and beyond. - _, err = r.ReadAt(buf6, 0) - if err != nil { - if err != io.EOF { - t.Fatal("Error:", err, len(buf6)) - } - } - err = c.RemoveObject(bucketName, objectName) - if err != nil { - t.Fatal("Error: ", err) - } - err = c.RemoveBucket(bucketName) - if err != nil { - t.Fatal("Error:", err) - } -} - -// Tests copy object -func TestCopyObjectV2(t *testing.T) { - if testing.Short() { - t.Skip("Skipping functional tests for short runs") - } - // Seed random based on current time. - rand.Seed(time.Now().Unix()) - - // Instantiate new minio client object - c, err := NewV2( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableSecurity)), - ) - if err != nil { - t.Fatal("Error:", err) - } - - // Enable tracing, write to stderr. - // c.TraceOn(os.Stderr) - - // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") - - // Generate a new random bucket name. - bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") - - // Make a new bucket in 'us-east-1' (source bucket). - err = c.MakeBucket(bucketName, "us-east-1") - if err != nil { - t.Fatal("Error:", err, bucketName) - } - - // Make a new bucket in 'us-east-1' (destination bucket). - err = c.MakeBucket(bucketName+"-copy", "us-east-1") - if err != nil { - t.Fatal("Error:", err, bucketName+"-copy") - } - - // Generate data more than 32K - buf := bytes.Repeat([]byte("9"), rand.Intn(1<<20)+32*1024) - - // Save the data - objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") - n, err := c.PutObject(bucketName, objectName, bytes.NewReader(buf), "binary/octet-stream") - if err != nil { - t.Fatal("Error:", err, bucketName, objectName) - } - - if n != int64(len(buf)) { - t.Fatalf("Error: number of bytes does not match want %v, got %v", - len(buf), n) - } - - // Set copy conditions. - copyConds := CopyConditions{} - err = copyConds.SetModified(time.Date(2014, time.April, 0, 0, 0, 0, 0, time.UTC)) - if err != nil { - t.Fatal("Error:", err) - } - - // Copy source. - copySource := bucketName + "/" + objectName - - // Perform the Copy - err = c.CopyObject(bucketName+"-copy", objectName+"-copy", copySource, copyConds) - if err != nil { - t.Fatal("Error:", err, bucketName+"-copy", objectName+"-copy") - } - - // Source object - reader, err := c.GetObject(bucketName, objectName) - if err != nil { - t.Fatal("Error:", err) - } - // Destination object - readerCopy, err := c.GetObject(bucketName+"-copy", objectName+"-copy") - if err != nil { - t.Fatal("Error:", err) - } - // Check the various fields of source object against destination object. - objInfo, err := reader.Stat() - if err != nil { - t.Fatal("Error:", err) - } - objInfoCopy, err := readerCopy.Stat() - if err != nil { - t.Fatal("Error:", err) - } - if objInfo.Size != objInfoCopy.Size { - t.Fatalf("Error: number of bytes does not match, want %v, got %v\n", - objInfo.Size, objInfoCopy.Size) - } - - // Remove all objects and buckets - err = c.RemoveObject(bucketName, objectName) - if err != nil { - t.Fatal("Error:", err) - } - - err = c.RemoveObject(bucketName+"-copy", objectName+"-copy") - if err != nil { - t.Fatal("Error:", err) - } - - err = c.RemoveBucket(bucketName) - if err != nil { - t.Fatal("Error:", err) - } - - err = c.RemoveBucket(bucketName + "-copy") - if err != nil { - t.Fatal("Error:", err) - } -} - -// Tests comprehensive list of all methods. -func TestFunctionalV2(t *testing.T) { - if testing.Short() { - t.Skip("skipping functional tests for the short runs") - } - - // Seed random based on current time. - rand.Seed(time.Now().Unix()) - - c, err := NewV2( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableSecurity)), - ) - if err != nil { - t.Fatal("Error:", err) - } - - // Enable to debug - // c.TraceOn(os.Stderr) - - // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") - - // Generate a new random bucket name. - bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") - - // Make a new bucket. - err = c.MakeBucket(bucketName, "us-east-1") - if err != nil { - t.Fatal("Error:", err, bucketName) - } - - // Generate a random file name. - fileName := randString(60, rand.NewSource(time.Now().UnixNano()), "") - file, err := os.Create(fileName) - if err != nil { - t.Fatal("Error:", err) - } - for i := 0; i < 3; i++ { - buf := make([]byte, rand.Intn(1<<19)) - _, err = file.Write(buf) - if err != nil { - t.Fatal("Error:", err) - } - } - file.Close() - - // Verify if bucket exits and you have access. - var exists bool - exists, err = c.BucketExists(bucketName) - if err != nil { - t.Fatal("Error:", err, bucketName) - } - if !exists { - t.Fatal("Error: could not find ", bucketName) - } - - // Make the bucket 'public read/write'. - err = c.SetBucketPolicy(bucketName, "", policy.BucketPolicyReadWrite) - if err != nil { - t.Fatal("Error:", err) - } - - // List all buckets. - buckets, err := c.ListBuckets() - if len(buckets) == 0 { - t.Fatal("Error: list buckets cannot be empty", buckets) - } - if err != nil { - t.Fatal("Error:", err) - } - - // Verify if previously created bucket is listed in list buckets. - bucketFound := false - for _, bucket := range buckets { - if bucket.Name == bucketName { - bucketFound = true - } - } - - // If bucket not found error out. - if !bucketFound { - t.Fatal("Error: bucket ", bucketName, "not found") - } - - objectName := bucketName + "unique" - - // Generate data - buf := bytes.Repeat([]byte("n"), rand.Intn(1<<19)) - - n, err := c.PutObject(bucketName, objectName, bytes.NewReader(buf), "") - if err != nil { - t.Fatal("Error: ", err) - } - if n != int64(len(buf)) { - t.Fatal("Error: bad length ", n, len(buf)) - } - - n, err = c.PutObject(bucketName, objectName+"-nolength", bytes.NewReader(buf), "binary/octet-stream") - if err != nil { - t.Fatal("Error:", err, bucketName, objectName+"-nolength") - } - - if n != int64(len(buf)) { - t.Fatalf("Error: number of bytes does not match, want %v, got %v\n", len(buf), n) - } - - // Instantiate a done channel to close all listing. - doneCh := make(chan struct{}) - defer close(doneCh) - - objFound := false - isRecursive := true // Recursive is true. - for obj := range c.ListObjects(bucketName, objectName, isRecursive, doneCh) { - if obj.Key == objectName { - objFound = true - break - } - } - if !objFound { - t.Fatal("Error: object " + objectName + " not found.") - } - - objFound = false - isRecursive = true // Recursive is true. - for obj := range c.ListObjects(bucketName, objectName, isRecursive, doneCh) { - if obj.Key == objectName { - objFound = true - break - } - } - if !objFound { - t.Fatal("Error: object " + objectName + " not found.") - } - - incompObjNotFound := true - for objIncompl := range c.ListIncompleteUploads(bucketName, objectName, isRecursive, doneCh) { - if objIncompl.Key != "" { - incompObjNotFound = false - break - } - } - if !incompObjNotFound { - t.Fatal("Error: unexpected dangling incomplete upload found.") - } - - newReader, err := c.GetObject(bucketName, objectName) - if err != nil { - t.Fatal("Error: ", err) - } - - newReadBytes, err := ioutil.ReadAll(newReader) - if err != nil { - t.Fatal("Error: ", err) - } - - if !bytes.Equal(newReadBytes, buf) { - t.Fatal("Error: bytes mismatch.") - } - - err = c.FGetObject(bucketName, objectName, fileName+"-f") - if err != nil { - t.Fatal("Error: ", err) - } - - // Generate presigned GET object url. - presignedGetURL, err := c.PresignedGetObject(bucketName, objectName, 3600*time.Second, nil) - if err != nil { - t.Fatal("Error: ", err) - } - // Verify if presigned url works. - resp, err := http.Get(presignedGetURL.String()) - if err != nil { - t.Fatal("Error: ", err) - } - if resp.StatusCode != http.StatusOK { - t.Fatal("Error: ", resp.Status) - } - newPresignedBytes, err := ioutil.ReadAll(resp.Body) - if err != nil { - t.Fatal("Error: ", err) - } - if !bytes.Equal(newPresignedBytes, buf) { - t.Fatal("Error: bytes mismatch.") - } - - // Set request parameters. - reqParams := make(url.Values) - reqParams.Set("response-content-disposition", "attachment; filename=\"test.txt\"") - // Generate presigned GET object url. - presignedGetURL, err = c.PresignedGetObject(bucketName, objectName, 3600*time.Second, reqParams) - if err != nil { - t.Fatal("Error: ", err) - } - // Verify if presigned url works. - resp, err = http.Get(presignedGetURL.String()) - if err != nil { - t.Fatal("Error: ", err) - } - if resp.StatusCode != http.StatusOK { - t.Fatal("Error: ", resp.Status) - } - newPresignedBytes, err = ioutil.ReadAll(resp.Body) - if err != nil { - t.Fatal("Error: ", err) - } - if !bytes.Equal(newPresignedBytes, buf) { - t.Fatal("Error: bytes mismatch for presigned GET url.") - } - // Verify content disposition. - if resp.Header.Get("Content-Disposition") != "attachment; filename=\"test.txt\"" { - t.Fatalf("Error: wrong Content-Disposition received %s", resp.Header.Get("Content-Disposition")) - } - - presignedPutURL, err := c.PresignedPutObject(bucketName, objectName+"-presigned", 3600*time.Second) - if err != nil { - t.Fatal("Error: ", err) - } - // Generate data more than 32K - buf = bytes.Repeat([]byte("1"), rand.Intn(1<<20)+32*1024) - - req, err := http.NewRequest("PUT", presignedPutURL.String(), bytes.NewReader(buf)) - if err != nil { - t.Fatal("Error: ", err) - } - httpClient := &http.Client{ - // Setting a sensible time out of 30secs to wait for response - // headers. Request is pro-actively cancelled after 30secs - // with no response. - Timeout: 30 * time.Second, - Transport: http.DefaultTransport, - } - resp, err = httpClient.Do(req) - if err != nil { - t.Fatal("Error: ", err) - } - - newReader, err = c.GetObject(bucketName, objectName+"-presigned") - if err != nil { - t.Fatal("Error: ", err) - } - - newReadBytes, err = ioutil.ReadAll(newReader) - if err != nil { - t.Fatal("Error: ", err) - } - - if !bytes.Equal(newReadBytes, buf) { - t.Fatal("Error: bytes mismatch.") - } - - err = c.RemoveObject(bucketName, objectName) - if err != nil { - t.Fatal("Error: ", err) - } - err = c.RemoveObject(bucketName, objectName+"-f") - if err != nil { - t.Fatal("Error: ", err) - } - err = c.RemoveObject(bucketName, objectName+"-nolength") - if err != nil { - t.Fatal("Error: ", err) - } - err = c.RemoveObject(bucketName, objectName+"-presigned") - if err != nil { - t.Fatal("Error: ", err) - } - err = c.RemoveBucket(bucketName) - if err != nil { - t.Fatal("Error:", err) - } - err = c.RemoveBucket(bucketName) - if err == nil { - t.Fatal("Error:") - } - if err.Error() != "The specified bucket does not exist" { - t.Fatal("Error: ", err) - } - if err = os.Remove(fileName); err != nil { - t.Fatal("Error: ", err) - } - if err = os.Remove(fileName + "-f"); err != nil { - t.Fatal("Error: ", err) - } -} diff --git a/vendor/src/github.com/minio/minio-go/api_functional_v4_test.go b/vendor/src/github.com/minio/minio-go/api_functional_v4_test.go deleted file mode 100644 index f54d76623..000000000 --- a/vendor/src/github.com/minio/minio-go/api_functional_v4_test.go +++ /dev/null @@ -1,2385 +0,0 @@ -/* - * Minio Go Library for Amazon S3 Compatible Cloud Storage (C) 2015 Minio, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package minio - -import ( - "bytes" - "encoding/hex" - "errors" - "fmt" - "io" - "io/ioutil" - "math/rand" - "net/http" - "net/url" - "os" - "strconv" - "strings" - "testing" - "time" - - "github.com/minio/minio-go/pkg/encrypt" - "github.com/minio/minio-go/pkg/policy" -) - -const letterBytes = "abcdefghijklmnopqrstuvwxyz01234569" -const ( - letterIdxBits = 6 // 6 bits to represent a letter index - letterIdxMask = 1<= 0; { - if remain == 0 { - cache, remain = src.Int63(), letterIdxMax - } - if idx := int(cache & letterIdxMask); idx < len(letterBytes) { - b[i] = letterBytes[idx] - i-- - } - cache >>= letterIdxBits - remain-- - } - return prefix + string(b[0:30-len(prefix)]) -} - -// Tests bucket re-create errors. -func TestMakeBucketError(t *testing.T) { - if testing.Short() { - t.Skip("skipping functional tests for short runs") - } - if os.Getenv(serverEndpoint) != "s3.amazonaws.com" { - t.Skip("skipping region functional tests for non s3 runs") - } - - // Seed random based on current time. - rand.Seed(time.Now().Unix()) - - // Instantiate new minio client object. - c, err := New( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableSecurity)), - ) - if err != nil { - t.Fatal("Error:", err) - } - - // Enable tracing, write to stderr. - // c.TraceOn(os.Stderr) - - // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") - - // Generate a new random bucket name. - bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") - - // Make a new bucket in 'eu-central-1'. - if err = c.MakeBucket(bucketName, "eu-central-1"); err != nil { - t.Fatal("Error:", err, bucketName) - } - if err = c.MakeBucket(bucketName, "eu-central-1"); err == nil { - t.Fatal("Error: make bucket should should fail for", bucketName) - } - // Verify valid error response from server. - if ToErrorResponse(err).Code != "BucketAlreadyExists" && - ToErrorResponse(err).Code != "BucketAlreadyOwnedByYou" { - t.Fatal("Error: Invalid error returned by server", err) - } - if err = c.RemoveBucket(bucketName); err != nil { - t.Fatal("Error:", err, bucketName) - } - if err = c.MakeBucket(bucketName+"..-1", "eu-central-1"); err == nil { - t.Fatal("Error:", err, bucketName+"..-1") - } - // Verify valid error response. - if ToErrorResponse(err).Code != "InvalidBucketName" { - t.Fatal("Error: Invalid error returned by server", err) - } - if err = c.MakeBucket(bucketName+"AAA-1", "eu-central-1"); err == nil { - t.Fatal("Error:", err, bucketName+"..-1") - } - // Verify valid error response. - if ToErrorResponse(err).Code != "InvalidBucketName" { - t.Fatal("Error: Invalid error returned by server", err) - } -} - -// Tests various bucket supported formats. -func TestMakeBucketRegions(t *testing.T) { - if testing.Short() { - t.Skip("skipping functional tests for short runs") - } - if os.Getenv(serverEndpoint) != "s3.amazonaws.com" { - t.Skip("skipping region functional tests for non s3 runs") - } - - // Seed random based on current time. - rand.Seed(time.Now().Unix()) - - // Instantiate new minio client object. - c, err := New( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableSecurity)), - ) - if err != nil { - t.Fatal("Error:", err) - } - - // Enable tracing, write to stderr. - // c.TraceOn(os.Stderr) - - // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") - - // Generate a new random bucket name. - bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") - - // Make a new bucket in 'eu-central-1'. - if err = c.MakeBucket(bucketName, "eu-central-1"); err != nil { - t.Fatal("Error:", err, bucketName) - } - - if err = c.RemoveBucket(bucketName); err != nil { - t.Fatal("Error:", err, bucketName) - } - - // Make a new bucket with '.' in its name, in 'us-west-2'. This - // request is internally staged into a path style instead of - // virtual host style. - if err = c.MakeBucket(bucketName+".withperiod", "us-west-2"); err != nil { - t.Fatal("Error:", err, bucketName+".withperiod") - } - - // Remove the newly created bucket. - if err = c.RemoveBucket(bucketName + ".withperiod"); err != nil { - t.Fatal("Error:", err, bucketName+".withperiod") - } -} - -// Test PutObject using a large data to trigger multipart readat -func TestPutObjectReadAt(t *testing.T) { - if testing.Short() { - t.Skip("skipping functional tests for short runs") - } - - // Seed random based on current time. - rand.Seed(time.Now().Unix()) - - // Instantiate new minio client object. - c, err := New( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableSecurity)), - ) - if err != nil { - t.Fatal("Error:", err) - } - - // Enable tracing, write to stderr. - // c.TraceOn(os.Stderr) - - // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") - - // Generate a new random bucket name. - bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") - - // Make a new bucket. - err = c.MakeBucket(bucketName, "us-east-1") - if err != nil { - t.Fatal("Error:", err, bucketName) - } - - // Generate data using 4 parts so that all 3 'workers' are utilized and a part is leftover. - // Use different data for each part for multipart tests to ensure part order at the end. - var buf []byte - for i := 0; i < 4; i++ { - buf = append(buf, bytes.Repeat([]byte(string('a'+i)), minPartSize)...) - } - - // Save the data - objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") - // Object content type - objectContentType := "binary/octet-stream" - - n, err := c.PutObject(bucketName, objectName, bytes.NewReader(buf), objectContentType) - if err != nil { - t.Fatal("Error:", err, bucketName, objectName) - } - - if n != int64(len(buf)) { - t.Fatalf("Error: number of bytes does not match, want %v, got %v\n", len(buf), n) - } - - // Read the data back - r, err := c.GetObject(bucketName, objectName) - if err != nil { - t.Fatal("Error:", err, bucketName, objectName) - } - - st, err := r.Stat() - if err != nil { - t.Fatal("Error:", err, bucketName, objectName) - } - if st.Size != int64(len(buf)) { - t.Fatalf("Error: number of bytes in stat does not match, want %v, got %v\n", - len(buf), st.Size) - } - if st.ContentType != objectContentType { - t.Fatalf("Error: Content types don't match, expected: %+v, found: %+v\n", objectContentType, st.ContentType) - } - if err := r.Close(); err != nil { - t.Fatal("Error:", err) - } - if err := r.Close(); err == nil { - t.Fatal("Error: object is already closed, should return error") - } - - err = c.RemoveObject(bucketName, objectName) - if err != nil { - t.Fatal("Error: ", err) - } - err = c.RemoveBucket(bucketName) - if err != nil { - t.Fatal("Error:", err) - } -} - -// Test PutObject using a large data to trigger multipart readat -func TestPutObjectWithMetadata(t *testing.T) { - if testing.Short() { - t.Skip("skipping functional tests for short runs") - } - - // Seed random based on current time. - rand.Seed(time.Now().Unix()) - - // Instantiate new minio client object. - c, err := New( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableSecurity)), - ) - if err != nil { - t.Fatal("Error:", err) - } - - // Enable tracing, write to stderr. - // c.TraceOn(os.Stderr) - - // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") - - // Generate a new random bucket name. - bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") - - // Make a new bucket. - err = c.MakeBucket(bucketName, "us-east-1") - if err != nil { - t.Fatal("Error:", err, bucketName) - } - - // Generate data using 2 parts - // Use different data in each part for multipart tests to ensure part order at the end. - var buf []byte - for i := 0; i < 2; i++ { - buf = append(buf, bytes.Repeat([]byte(string('a'+i)), minPartSize)...) - } - - // Save the data - objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") - - // Object custom metadata - customContentType := "custom/contenttype" - - n, err := c.PutObjectWithMetadata(bucketName, objectName, bytes.NewReader(buf), map[string][]string{"Content-Type": {customContentType}}, nil) - if err != nil { - t.Fatal("Error:", err, bucketName, objectName) - } - - if n != int64(len(buf)) { - t.Fatalf("Error: number of bytes does not match, want %v, got %v\n", len(buf), n) - } - - // Read the data back - r, err := c.GetObject(bucketName, objectName) - if err != nil { - t.Fatal("Error:", err, bucketName, objectName) - } - - st, err := r.Stat() - if err != nil { - t.Fatal("Error:", err, bucketName, objectName) - } - if st.Size != int64(len(buf)) { - t.Fatalf("Error: number of bytes in stat does not match, want %v, got %v\n", - len(buf), st.Size) - } - if st.ContentType != customContentType { - t.Fatalf("Error: Expected and found content types do not match, want %v, got %v\n", - customContentType, st.ContentType) - } - if err := r.Close(); err != nil { - t.Fatal("Error:", err) - } - if err := r.Close(); err == nil { - t.Fatal("Error: object is already closed, should return error") - } - - err = c.RemoveObject(bucketName, objectName) - if err != nil { - t.Fatal("Error: ", err) - } - err = c.RemoveBucket(bucketName) - if err != nil { - t.Fatal("Error:", err) - } -} - -// Test put object with streaming signature. -func TestPutObjectStreaming(t *testing.T) { - if testing.Short() { - t.Skip("skipping function tests for short runs") - } - - // Seed random based on current time. - rand.Seed(time.Now().Unix()) - - // Instantiate new minio client object. - c, err := NewV4( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableSecurity)), - ) - if err != nil { - t.Fatal("Error:", err) - } - - // Enable tracing, write to stderr. - // c.TraceOn(os.Stderr) - - // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") - - // Generate a new random bucket name. - bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), - "minio-go-test") - - // Make a new bucket. - err = c.MakeBucket(bucketName, "us-east-1") - if err != nil { - t.Fatal("Error:", err, bucketName) - } - - // Upload an object. - sizes := []int64{0, 64*1024 - 1, 64 * 1024} - objectName := "test-object" - for i, size := range sizes { - data := bytes.Repeat([]byte("a"), int(size)) - n, err := c.PutObjectStreaming(bucketName, objectName, bytes.NewReader(data)) - if err != nil { - t.Fatalf("Test %d Error: %v %s %s", i+1, err, bucketName, objectName) - } - - if n != size { - t.Errorf("Test %d Expected upload object size %d but got %d", i+1, size, n) - } - } - - // Remove the object. - err = c.RemoveObject(bucketName, objectName) - if err != nil { - t.Fatal("Error:", err) - } - - // Remove the bucket. - err = c.RemoveBucket(bucketName) - if err != nil { - t.Fatal("Error:", err) - } -} - -// Test listing partially uploaded objects. -func TestListPartiallyUploaded(t *testing.T) { - if testing.Short() { - t.Skip("skipping function tests for short runs") - } - - // Seed random based on current time. - rand.Seed(time.Now().Unix()) - - // Instantiate new minio client object. - c, err := New( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableSecurity)), - ) - if err != nil { - t.Fatal("Error:", err) - } - - // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") - - // Enable tracing, write to stdout. - // c.TraceOn(os.Stderr) - - // Generate a new random bucket name. - bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") - - // Make a new bucket. - err = c.MakeBucket(bucketName, "us-east-1") - if err != nil { - t.Fatal("Error:", err, bucketName) - } - - r := bytes.NewReader(bytes.Repeat([]byte("0"), minPartSize*2)) - - reader, writer := io.Pipe() - go func() { - i := 0 - for i < 25 { - _, cerr := io.CopyN(writer, r, (minPartSize*2)/25) - if cerr != nil { - t.Fatal("Error:", cerr, bucketName) - } - i++ - r.Seek(0, 0) - } - writer.CloseWithError(errors.New("proactively closed to be verified later")) - }() - - objectName := bucketName + "-resumable" - _, err = c.PutObject(bucketName, objectName, reader, "application/octet-stream") - if err == nil { - t.Fatal("Error: PutObject should fail.") - } - if err.Error() != "proactively closed to be verified later" { - t.Fatal("Error:", err) - } - - doneCh := make(chan struct{}) - defer close(doneCh) - isRecursive := true - multiPartObjectCh := c.ListIncompleteUploads(bucketName, objectName, isRecursive, doneCh) - for multiPartObject := range multiPartObjectCh { - if multiPartObject.Err != nil { - t.Fatalf("Error: Error when listing incomplete upload") - } - } - - err = c.RemoveBucket(bucketName) - if err != nil { - t.Fatal("Error:", err) - } -} - -// Test get object seeker from the end, using whence set to '2'. -func TestGetOjectSeekEnd(t *testing.T) { - if testing.Short() { - t.Skip("skipping functional tests for short runs") - } - - // Seed random based on current time. - rand.Seed(time.Now().Unix()) - - // Instantiate new minio client object. - c, err := New( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableSecurity)), - ) - if err != nil { - t.Fatal("Error:", err) - } - - // Enable tracing, write to stderr. - // c.TraceOn(os.Stderr) - - // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") - - // Generate a new random bucket name. - bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") - - // Make a new bucket. - err = c.MakeBucket(bucketName, "us-east-1") - if err != nil { - t.Fatal("Error:", err, bucketName) - } - - // Generate data more than 32K - buf := bytes.Repeat([]byte("1"), rand.Intn(1<<20)+32*1024) - // Save the data - objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") - n, err := c.PutObject(bucketName, objectName, bytes.NewReader(buf), "binary/octet-stream") - if err != nil { - t.Fatal("Error:", err, bucketName, objectName) - } - - if n != int64(len(buf)) { - t.Fatalf("Error: number of bytes does not match, want %v, got %v\n", len(buf), n) - } - - // Read the data back - r, err := c.GetObject(bucketName, objectName) - if err != nil { - t.Fatal("Error:", err, bucketName, objectName) - } - - st, err := r.Stat() - if err != nil { - t.Fatal("Error:", err, bucketName, objectName) - } - if st.Size != int64(len(buf)) { - t.Fatalf("Error: number of bytes in stat does not match, want %v, got %v\n", - len(buf), st.Size) - } - - pos, err := r.Seek(-100, 2) - if err != nil { - t.Fatal("Error:", err, bucketName, objectName) - } - if pos != st.Size-100 { - t.Fatalf("Expected %d, got %d instead", pos, st.Size-100) - } - buf2 := make([]byte, 100) - m, err := io.ReadFull(r, buf2) - if err != nil { - t.Fatal("Error: reading through io.ReadFull", err, bucketName, objectName) - } - if m != len(buf2) { - t.Fatalf("Expected %d bytes, got %d", len(buf2), m) - } - hexBuf1 := fmt.Sprintf("%02x", buf[len(buf)-100:]) - hexBuf2 := fmt.Sprintf("%02x", buf2[:m]) - if hexBuf1 != hexBuf2 { - t.Fatalf("Expected %s, got %s instead", hexBuf1, hexBuf2) - } - pos, err = r.Seek(-100, 2) - if err != nil { - t.Fatal("Error:", err, bucketName, objectName) - } - if pos != st.Size-100 { - t.Fatalf("Expected %d, got %d instead", pos, st.Size-100) - } - if err = r.Close(); err != nil { - t.Fatal("Error:", err, bucketName, objectName) - } -} - -// Test get object reader to not throw error on being closed twice. -func TestGetObjectClosedTwice(t *testing.T) { - if testing.Short() { - t.Skip("skipping functional tests for short runs") - } - - // Seed random based on current time. - rand.Seed(time.Now().Unix()) - - // Instantiate new minio client object. - c, err := New( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableSecurity)), - ) - if err != nil { - t.Fatal("Error:", err) - } - - // Enable tracing, write to stderr. - // c.TraceOn(os.Stderr) - - // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") - - // Generate a new random bucket name. - bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") - - // Make a new bucket. - err = c.MakeBucket(bucketName, "us-east-1") - if err != nil { - t.Fatal("Error:", err, bucketName) - } - - // Generate data more than 32K - buf := bytes.Repeat([]byte("1"), rand.Intn(1<<20)+32*1024) - - // Save the data - objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") - n, err := c.PutObject(bucketName, objectName, bytes.NewReader(buf), "binary/octet-stream") - if err != nil { - t.Fatal("Error:", err, bucketName, objectName) - } - - if n != int64(len(buf)) { - t.Fatalf("Error: number of bytes does not match, want %v, got %v\n", len(buf), n) - } - - // Read the data back - r, err := c.GetObject(bucketName, objectName) - if err != nil { - t.Fatal("Error:", err, bucketName, objectName) - } - - st, err := r.Stat() - if err != nil { - t.Fatal("Error:", err, bucketName, objectName) - } - if st.Size != int64(len(buf)) { - t.Fatalf("Error: number of bytes in stat does not match, want %v, got %v\n", - len(buf), st.Size) - } - if err := r.Close(); err != nil { - t.Fatal("Error:", err) - } - if err := r.Close(); err == nil { - t.Fatal("Error: object is already closed, should return error") - } - - err = c.RemoveObject(bucketName, objectName) - if err != nil { - t.Fatal("Error: ", err) - } - err = c.RemoveBucket(bucketName) - if err != nil { - t.Fatal("Error:", err) - } -} - -// Test removing multiple objects with Remove API -func TestRemoveMultipleObjects(t *testing.T) { - if testing.Short() { - t.Skip("skipping function tests for short runs") - } - - // Seed random based on current time. - rand.Seed(time.Now().Unix()) - - // Instantiate new minio client object. - c, err := New( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableSecurity)), - ) - - if err != nil { - t.Fatal("Error:", err) - } - - // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") - - // Enable tracing, write to stdout. - // c.TraceOn(os.Stderr) - - // Generate a new random bucket name. - bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") - - // Make a new bucket. - err = c.MakeBucket(bucketName, "us-east-1") - if err != nil { - t.Fatal("Error:", err, bucketName) - } - - r := bytes.NewReader(bytes.Repeat([]byte("a"), 8)) - - // Multi remove of 1100 objects - nrObjects := 1100 - - objectsCh := make(chan string) - - go func() { - defer close(objectsCh) - // Upload objects and send them to objectsCh - for i := 0; i < nrObjects; i++ { - objectName := "sample" + strconv.Itoa(i) + ".txt" - _, err = c.PutObject(bucketName, objectName, r, "application/octet-stream") - if err != nil { - t.Error("Error: PutObject shouldn't fail.", err) - continue - } - objectsCh <- objectName - } - }() - - // Call RemoveObjects API - errorCh := c.RemoveObjects(bucketName, objectsCh) - - // Check if errorCh doesn't receive any error - select { - case r, more := <-errorCh: - if more { - t.Fatalf("Unexpected error, objName(%v) err(%v)", r.ObjectName, r.Err) - } - } - - // Clean the bucket created by the test - err = c.RemoveBucket(bucketName) - if err != nil { - t.Fatal("Error:", err) - } -} - -// Tests removing partially uploaded objects. -func TestRemovePartiallyUploaded(t *testing.T) { - if testing.Short() { - t.Skip("skipping function tests for short runs") - } - - // Seed random based on current time. - rand.Seed(time.Now().Unix()) - - // Instantiate new minio client object. - c, err := New( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableSecurity)), - ) - if err != nil { - t.Fatal("Error:", err) - } - - // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") - - // Enable tracing, write to stdout. - // c.TraceOn(os.Stderr) - - // Generate a new random bucket name. - bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") - - // Make a new bucket. - err = c.MakeBucket(bucketName, "us-east-1") - if err != nil { - t.Fatal("Error:", err, bucketName) - } - - r := bytes.NewReader(bytes.Repeat([]byte("a"), 128*1024)) - - reader, writer := io.Pipe() - go func() { - i := 0 - for i < 25 { - _, cerr := io.CopyN(writer, r, 128*1024) - if cerr != nil { - t.Fatal("Error:", cerr, bucketName) - } - i++ - r.Seek(0, 0) - } - writer.CloseWithError(errors.New("proactively closed to be verified later")) - }() - - objectName := bucketName + "-resumable" - _, err = c.PutObject(bucketName, objectName, reader, "application/octet-stream") - if err == nil { - t.Fatal("Error: PutObject should fail.") - } - if err.Error() != "proactively closed to be verified later" { - t.Fatal("Error:", err) - } - err = c.RemoveIncompleteUpload(bucketName, objectName) - if err != nil { - t.Fatal("Error:", err) - } - err = c.RemoveBucket(bucketName) - if err != nil { - t.Fatal("Error:", err) - } -} - -// Tests FPutObject of a big file to trigger multipart -func TestFPutObjectMultipart(t *testing.T) { - if testing.Short() { - t.Skip("skipping functional tests for short runs") - } - - // Seed random based on current time. - rand.Seed(time.Now().Unix()) - - // Instantiate new minio client object. - c, err := New( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableSecurity)), - ) - if err != nil { - t.Fatal("Error:", err) - } - - // Enable tracing, write to stderr. - // c.TraceOn(os.Stderr) - - // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") - - // Generate a new random bucket name. - bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") - - // Make a new bucket. - err = c.MakeBucket(bucketName, "us-east-1") - if err != nil { - t.Fatal("Error:", err, bucketName) - } - - // Make a temp file with minPartSize*4 bytes of data. - file, err := ioutil.TempFile(os.TempDir(), "FPutObjectTest") - if err != nil { - t.Fatal("Error:", err) - } - - // Upload 4 parts to utilize all 3 'workers' in multipart and still have a part to upload. - var buffer []byte - for i := 0; i < 4; i++ { - buffer = append(buffer, bytes.Repeat([]byte(string('a'+i)), minPartSize)...) - } - - size, err := file.Write(buffer) - if err != nil { - t.Fatal("Error:", err) - } - if size != minPartSize*4 { - t.Fatalf("Error: number of bytes does not match, want %v, got %v\n", minPartSize*4, size) - } - - // Close the file pro-actively for windows. - err = file.Close() - if err != nil { - t.Fatal("Error:", err) - } - - // Set base object name - objectName := bucketName + "FPutObject" - objectContentType := "testapplication/octet-stream" - - // Perform standard FPutObject with contentType provided (Expecting application/octet-stream) - n, err := c.FPutObject(bucketName, objectName+"-standard", file.Name(), objectContentType) - if err != nil { - t.Fatal("Error:", err) - } - if n != int64(minPartSize*4) { - t.Fatalf("Error: number of bytes does not match, want %v, got %v\n", minPartSize*4, n) - } - - r, err := c.GetObject(bucketName, objectName+"-standard") - if err != nil { - t.Fatalf("Unexpected error: %v\n", err) - } - objInfo, err := r.Stat() - if err != nil { - t.Fatalf("Unexpected error: %v\n", err) - } - if objInfo.Size != minPartSize*4 { - t.Fatalf("Error: number of bytes does not match, want %v, got %v\n", minPartSize*4, n) - } - if objInfo.ContentType != objectContentType { - t.Fatalf("Error: Content types don't match, want %v, got %v\n", objectContentType, objInfo.ContentType) - } - - // Remove all objects and bucket and temp file - err = c.RemoveObject(bucketName, objectName+"-standard") - if err != nil { - t.Fatal("Error: ", err) - } - - err = c.RemoveBucket(bucketName) - if err != nil { - t.Fatal("Error:", err) - } -} - -// Tests FPutObject hidden contentType setting -func TestFPutObject(t *testing.T) { - if testing.Short() { - t.Skip("skipping functional tests for short runs") - } - - // Seed random based on current time. - rand.Seed(time.Now().Unix()) - - // Instantiate new minio client object. - c, err := New( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableSecurity)), - ) - if err != nil { - t.Fatal("Error:", err) - } - - // Enable tracing, write to stderr. - // c.TraceOn(os.Stderr) - - // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") - - // Generate a new random bucket name. - bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") - - // Make a new bucket. - err = c.MakeBucket(bucketName, "us-east-1") - if err != nil { - t.Fatal("Error:", err, bucketName) - } - - // Make a temp file with minPartSize*4 bytes of data. - file, err := ioutil.TempFile(os.TempDir(), "FPutObjectTest") - if err != nil { - t.Fatal("Error:", err) - } - - // Upload 4 parts worth of data to use all 3 of multiparts 'workers' and have an extra part. - // Use different data in part for multipart tests to check parts are uploaded in correct order. - var buffer []byte - for i := 0; i < 4; i++ { - buffer = append(buffer, bytes.Repeat([]byte(string('a'+i)), minPartSize)...) - } - - // Write the data to the file. - size, err := file.Write(buffer) - if err != nil { - t.Fatal("Error:", err) - } - if size != minPartSize*4 { - t.Fatalf("Error: number of bytes does not match, want %v, got %v\n", minPartSize*4, size) - } - - // Close the file pro-actively for windows. - err = file.Close() - if err != nil { - t.Fatal("Error:", err) - } - - // Set base object name - objectName := bucketName + "FPutObject" - - // Perform standard FPutObject with contentType provided (Expecting application/octet-stream) - n, err := c.FPutObject(bucketName, objectName+"-standard", file.Name(), "application/octet-stream") - if err != nil { - t.Fatal("Error:", err) - } - if n != int64(minPartSize*4) { - t.Fatalf("Error: number of bytes does not match, want %v, got %v\n", minPartSize*4, n) - } - - // Perform FPutObject with no contentType provided (Expecting application/octet-stream) - n, err = c.FPutObject(bucketName, objectName+"-Octet", file.Name(), "") - if err != nil { - t.Fatal("Error:", err) - } - if n != int64(minPartSize*4) { - t.Fatalf("Error: number of bytes does not match, want %v, got %v\n", minPartSize*4, n) - } - - // Add extension to temp file name - fileName := file.Name() - err = os.Rename(file.Name(), fileName+".gtar") - if err != nil { - t.Fatal("Error:", err) - } - - // Perform FPutObject with no contentType provided (Expecting application/x-gtar) - n, err = c.FPutObject(bucketName, objectName+"-GTar", fileName+".gtar", "") - if err != nil { - t.Fatal("Error:", err) - } - if n != int64(minPartSize*4) { - t.Fatalf("Error: number of bytes does not match, want %v, got %v\n", minPartSize*4, n) - } - - // Check headers - rStandard, err := c.StatObject(bucketName, objectName+"-standard") - if err != nil { - t.Fatal("Error:", err, bucketName, objectName+"-standard") - } - if rStandard.ContentType != "application/octet-stream" { - t.Fatalf("Error: Content-Type headers mismatched, want %v, got %v\n", - "application/octet-stream", rStandard.ContentType) - } - - rOctet, err := c.StatObject(bucketName, objectName+"-Octet") - if err != nil { - t.Fatal("Error:", err, bucketName, objectName+"-Octet") - } - if rOctet.ContentType != "application/octet-stream" { - t.Fatalf("Error: Content-Type headers mismatched, want %v, got %v\n", - "application/octet-stream", rStandard.ContentType) - } - - rGTar, err := c.StatObject(bucketName, objectName+"-GTar") - if err != nil { - t.Fatal("Error:", err, bucketName, objectName+"-GTar") - } - if rGTar.ContentType != "application/x-gtar" { - t.Fatalf("Error: Content-Type headers mismatched, want %v, got %v\n", - "application/x-gtar", rStandard.ContentType) - } - - // Remove all objects and bucket and temp file - err = c.RemoveObject(bucketName, objectName+"-standard") - if err != nil { - t.Fatal("Error: ", err) - } - - err = c.RemoveObject(bucketName, objectName+"-Octet") - if err != nil { - t.Fatal("Error: ", err) - } - - err = c.RemoveObject(bucketName, objectName+"-GTar") - if err != nil { - t.Fatal("Error: ", err) - } - - err = c.RemoveBucket(bucketName) - if err != nil { - t.Fatal("Error:", err) - } - - err = os.Remove(fileName + ".gtar") - if err != nil { - t.Fatal("Error:", err) - } - -} - -// Tests get object ReaderSeeker interface methods. -func TestGetObjectReadSeekFunctional(t *testing.T) { - if testing.Short() { - t.Skip("skipping functional tests for short runs") - } - - // Seed random based on current time. - rand.Seed(time.Now().Unix()) - - // Instantiate new minio client object. - c, err := New( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableSecurity)), - ) - if err != nil { - t.Fatal("Error:", err) - } - - // Enable tracing, write to stderr. - // c.TraceOn(os.Stderr) - - // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") - - // Generate a new random bucket name. - bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") - - // Make a new bucket. - err = c.MakeBucket(bucketName, "us-east-1") - if err != nil { - t.Fatal("Error:", err, bucketName) - } - - // Generate data more than 32K - buf := bytes.Repeat([]byte("2"), rand.Intn(1<<20)+32*1024) - bufSize := len(buf) - - // Save the data - objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") - n, err := c.PutObject(bucketName, objectName, bytes.NewReader(buf), "binary/octet-stream") - if err != nil { - t.Fatal("Error:", err, bucketName, objectName) - } - - if n != int64(bufSize) { - t.Fatalf("Error: number of bytes does not match, want %v, got %v\n", len(buf), n) - } - - defer func() { - err = c.RemoveObject(bucketName, objectName) - if err != nil { - t.Fatal("Error: ", err) - } - err = c.RemoveBucket(bucketName) - if err != nil { - t.Fatal("Error:", err) - } - }() - - // Read the data back - r, err := c.GetObject(bucketName, objectName) - if err != nil { - t.Fatal("Error:", err, bucketName, objectName) - } - - st, err := r.Stat() - if err != nil { - t.Fatal("Error:", err, bucketName, objectName) - } - if st.Size != int64(bufSize) { - t.Fatalf("Error: number of bytes in stat does not match, want %v, got %v\n", - len(buf), st.Size) - } - - // This following function helps us to compare data from the reader after seek - // with the data from the original buffer - cmpData := func(r io.Reader, start, end int) { - if end-start == 0 { - return - } - buffer := bytes.NewBuffer([]byte{}) - if _, err := io.CopyN(buffer, r, int64(bufSize)); err != nil { - if err != io.EOF { - t.Fatal("Error:", err) - } - } - if !bytes.Equal(buf[start:end], buffer.Bytes()) { - t.Fatal("Error: Incorrect read bytes v/s original buffer.") - } - } - - // Generic seek error for errors other than io.EOF - seekErr := errors.New("seek error") - - testCases := []struct { - offset int64 - whence int - pos int64 - err error - shouldCmp bool - start int - end int - }{ - // Start from offset 0, fetch data and compare - {0, 0, 0, nil, true, 0, 0}, - // Start from offset 2048, fetch data and compare - {2048, 0, 2048, nil, true, 2048, bufSize}, - // Start from offset larger than possible - {int64(bufSize) + 1024, 0, 0, seekErr, false, 0, 0}, - // Move to offset 0 without comparing - {0, 0, 0, nil, false, 0, 0}, - // Move one step forward and compare - {1, 1, 1, nil, true, 1, bufSize}, - // Move larger than possible - {int64(bufSize), 1, 0, seekErr, false, 0, 0}, - // Provide negative offset with CUR_SEEK - {int64(-1), 1, 0, seekErr, false, 0, 0}, - // Test with whence SEEK_END and with positive offset - {1024, 2, int64(bufSize) - 1024, io.EOF, true, 0, 0}, - // Test with whence SEEK_END and with negative offset - {-1024, 2, int64(bufSize) - 1024, nil, true, bufSize - 1024, bufSize}, - // Test with whence SEEK_END and with large negative offset - {-int64(bufSize) * 2, 2, 0, seekErr, true, 0, 0}, - } - - for i, testCase := range testCases { - // Perform seek operation - n, err := r.Seek(testCase.offset, testCase.whence) - // We expect an error - if testCase.err == seekErr && err == nil { - t.Fatalf("Test %d, unexpected err value: expected: %v, found: %v", i+1, testCase.err, err) - } - // We expect a specific error - if testCase.err != seekErr && testCase.err != err { - t.Fatalf("Test %d, unexpected err value: expected: %v, found: %v", i+1, testCase.err, err) - } - // If we expect an error go to the next loop - if testCase.err != nil { - continue - } - // Check the returned seek pos - if n != testCase.pos { - t.Fatalf("Test %d, error: number of bytes seeked does not match, want %v, got %v\n", i+1, - testCase.pos, n) - } - // Compare only if shouldCmp is activated - if testCase.shouldCmp { - cmpData(r, testCase.start, testCase.end) - } - } -} - -// Tests get object ReaderAt interface methods. -func TestGetObjectReadAtFunctional(t *testing.T) { - if testing.Short() { - t.Skip("skipping functional tests for the short runs") - } - - // Seed random based on current time. - rand.Seed(time.Now().Unix()) - - // Instantiate new minio client object. - c, err := New( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableSecurity)), - ) - if err != nil { - t.Fatal("Error:", err) - } - - // Enable tracing, write to stderr. - // c.TraceOn(os.Stderr) - - // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") - - // Generate a new random bucket name. - bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") - - // Make a new bucket. - err = c.MakeBucket(bucketName, "us-east-1") - if err != nil { - t.Fatal("Error:", err, bucketName) - } - - // Generate data more than 32K - buf := bytes.Repeat([]byte("3"), rand.Intn(1<<20)+32*1024) - - // Save the data - objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") - n, err := c.PutObject(bucketName, objectName, bytes.NewReader(buf), "binary/octet-stream") - if err != nil { - t.Fatal("Error:", err, bucketName, objectName) - } - - if n != int64(len(buf)) { - t.Fatalf("Error: number of bytes does not match, want %v, got %v\n", len(buf), n) - } - - // read the data back - r, err := c.GetObject(bucketName, objectName) - if err != nil { - t.Fatal("Error:", err, bucketName, objectName) - } - offset := int64(2048) - - // read directly - buf1 := make([]byte, 512) - buf2 := make([]byte, 512) - buf3 := make([]byte, 512) - buf4 := make([]byte, 512) - - // Test readAt before stat is called. - m, err := r.ReadAt(buf1, offset) - if err != nil { - t.Fatal("Error:", err, len(buf1), offset) - } - if m != len(buf1) { - t.Fatalf("Error: ReadAt read shorter bytes before reaching EOF, want %v, got %v\n", m, len(buf1)) - } - if !bytes.Equal(buf1, buf[offset:offset+512]) { - t.Fatal("Error: Incorrect read between two ReadAt from same offset.") - } - offset += 512 - - st, err := r.Stat() - if err != nil { - t.Fatal("Error:", err, bucketName, objectName) - } - if st.Size != int64(len(buf)) { - t.Fatalf("Error: number of bytes in stat does not match, want %v, got %v\n", - len(buf), st.Size) - } - - m, err = r.ReadAt(buf2, offset) - if err != nil { - t.Fatal("Error:", err, st.Size, len(buf2), offset) - } - if m != len(buf2) { - t.Fatalf("Error: ReadAt read shorter bytes before reaching EOF, want %v, got %v\n", m, len(buf2)) - } - if !bytes.Equal(buf2, buf[offset:offset+512]) { - t.Fatal("Error: Incorrect read between two ReadAt from same offset.") - } - offset += 512 - m, err = r.ReadAt(buf3, offset) - if err != nil { - t.Fatal("Error:", err, st.Size, len(buf3), offset) - } - if m != len(buf3) { - t.Fatalf("Error: ReadAt read shorter bytes before reaching EOF, want %v, got %v\n", m, len(buf3)) - } - if !bytes.Equal(buf3, buf[offset:offset+512]) { - t.Fatal("Error: Incorrect read between two ReadAt from same offset.") - } - offset += 512 - m, err = r.ReadAt(buf4, offset) - if err != nil { - t.Fatal("Error:", err, st.Size, len(buf4), offset) - } - if m != len(buf4) { - t.Fatalf("Error: ReadAt read shorter bytes before reaching EOF, want %v, got %v\n", m, len(buf4)) - } - if !bytes.Equal(buf4, buf[offset:offset+512]) { - t.Fatal("Error: Incorrect read between two ReadAt from same offset.") - } - - buf5 := make([]byte, n) - // Read the whole object. - m, err = r.ReadAt(buf5, 0) - if err != nil { - if err != io.EOF { - t.Fatal("Error:", err, len(buf5)) - } - } - if m != len(buf5) { - t.Fatalf("Error: ReadAt read shorter bytes before reaching EOF, want %v, got %v\n", m, len(buf5)) - } - if !bytes.Equal(buf, buf5) { - t.Fatal("Error: Incorrect data read in GetObject, than what was previously upoaded.") - } - - buf6 := make([]byte, n+1) - // Read the whole object and beyond. - _, err = r.ReadAt(buf6, 0) - if err != nil { - if err != io.EOF { - t.Fatal("Error:", err, len(buf6)) - } - } - err = c.RemoveObject(bucketName, objectName) - if err != nil { - t.Fatal("Error: ", err) - } - err = c.RemoveBucket(bucketName) - if err != nil { - t.Fatal("Error:", err) - } -} - -// Test Presigned Post Policy -func TestPresignedPostPolicy(t *testing.T) { - if testing.Short() { - t.Skip("Skipping functional tests for short runs") - } - // Seed random based on current time. - rand.Seed(time.Now().Unix()) - - // Instantiate new minio client object - c, err := NewV4( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableSecurity)), - ) - if err != nil { - t.Fatal("Error:", err) - } - - // Enable tracing, write to stderr. - // c.TraceOn(os.Stderr) - - // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") - - // Generate a new random bucket name. - bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") - - // Make a new bucket in 'us-east-1' (source bucket). - err = c.MakeBucket(bucketName, "us-east-1") - if err != nil { - t.Fatal("Error:", err, bucketName) - } - - // Generate data more than 32K - buf := bytes.Repeat([]byte("4"), rand.Intn(1<<20)+32*1024) - - // Save the data - objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") - n, err := c.PutObject(bucketName, objectName, bytes.NewReader(buf), "binary/octet-stream") - if err != nil { - t.Fatal("Error:", err, bucketName, objectName) - } - - if n != int64(len(buf)) { - t.Fatalf("Error: number of bytes does not match want %v, got %v", - len(buf), n) - } - - policy := NewPostPolicy() - - if err := policy.SetBucket(""); err == nil { - t.Fatalf("Error: %s", err) - } - if err := policy.SetKey(""); err == nil { - t.Fatalf("Error: %s", err) - } - if err := policy.SetKeyStartsWith(""); err == nil { - t.Fatalf("Error: %s", err) - } - if err := policy.SetExpires(time.Date(1, time.January, 1, 0, 0, 0, 0, time.UTC)); err == nil { - t.Fatalf("Error: %s", err) - } - if err := policy.SetContentType(""); err == nil { - t.Fatalf("Error: %s", err) - } - if err := policy.SetContentLengthRange(1024*1024, 1024); err == nil { - t.Fatalf("Error: %s", err) - } - - policy.SetBucket(bucketName) - policy.SetKey(objectName) - policy.SetExpires(time.Now().UTC().AddDate(0, 0, 10)) // expires in 10 days - policy.SetContentType("image/png") - policy.SetContentLengthRange(1024, 1024*1024) - - _, _, err = c.PresignedPostPolicy(policy) - if err != nil { - t.Fatal("Error:", err) - } - - policy = NewPostPolicy() - - // Remove all objects and buckets - err = c.RemoveObject(bucketName, objectName) - if err != nil { - t.Fatal("Error:", err) - } - - err = c.RemoveBucket(bucketName) - if err != nil { - t.Fatal("Error:", err) - } -} - -// Tests copy object -func TestCopyObject(t *testing.T) { - if testing.Short() { - t.Skip("Skipping functional tests for short runs") - } - // Seed random based on current time. - rand.Seed(time.Now().Unix()) - - // Instantiate new minio client object - c, err := NewV4( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableSecurity)), - ) - if err != nil { - t.Fatal("Error:", err) - } - - // Enable tracing, write to stderr. - // c.TraceOn(os.Stderr) - - // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") - - // Generate a new random bucket name. - bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") - - // Make a new bucket in 'us-east-1' (source bucket). - err = c.MakeBucket(bucketName, "us-east-1") - if err != nil { - t.Fatal("Error:", err, bucketName) - } - - // Make a new bucket in 'us-east-1' (destination bucket). - err = c.MakeBucket(bucketName+"-copy", "us-east-1") - if err != nil { - t.Fatal("Error:", err, bucketName+"-copy") - } - - // Generate data more than 32K - buf := bytes.Repeat([]byte("5"), rand.Intn(1<<20)+32*1024) - - // Save the data - objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") - n, err := c.PutObject(bucketName, objectName, bytes.NewReader(buf), "binary/octet-stream") - if err != nil { - t.Fatal("Error:", err, bucketName, objectName) - } - - if n != int64(len(buf)) { - t.Fatalf("Error: number of bytes does not match want %v, got %v", - len(buf), n) - } - - r, err := c.GetObject(bucketName, objectName) - if err != nil { - t.Fatal("Error:", err) - } - // Check the various fields of source object against destination object. - objInfo, err := r.Stat() - if err != nil { - t.Fatal("Error:", err) - } - - // Set copy conditions. - copyConds := CopyConditions{} - - // Start by setting wrong conditions - err = copyConds.SetModified(time.Date(1, time.January, 1, 0, 0, 0, 0, time.UTC)) - if err == nil { - t.Fatal("Error:", err) - } - err = copyConds.SetUnmodified(time.Date(1, time.January, 1, 0, 0, 0, 0, time.UTC)) - if err == nil { - t.Fatal("Error:", err) - } - err = copyConds.SetMatchETag("") - if err == nil { - t.Fatal("Error:", err) - } - err = copyConds.SetMatchETagExcept("") - if err == nil { - t.Fatal("Error:", err) - } - - err = copyConds.SetModified(time.Date(2014, time.April, 0, 0, 0, 0, 0, time.UTC)) - if err != nil { - t.Fatal("Error:", err) - } - err = copyConds.SetMatchETag(objInfo.ETag) - if err != nil { - t.Fatal("Error:", err) - } - - // Copy source. - copySource := bucketName + "/" + objectName - - // Perform the Copy - err = c.CopyObject(bucketName+"-copy", objectName+"-copy", copySource, copyConds) - if err != nil { - t.Fatal("Error:", err, bucketName+"-copy", objectName+"-copy") - } - - // Source object - reader, err := c.GetObject(bucketName, objectName) - if err != nil { - t.Fatal("Error:", err) - } - // Destination object - readerCopy, err := c.GetObject(bucketName+"-copy", objectName+"-copy") - if err != nil { - t.Fatal("Error:", err) - } - // Check the various fields of source object against destination object. - objInfo, err = reader.Stat() - if err != nil { - t.Fatal("Error:", err) - } - objInfoCopy, err := readerCopy.Stat() - if err != nil { - t.Fatal("Error:", err) - } - if objInfo.Size != objInfoCopy.Size { - t.Fatalf("Error: number of bytes does not match, want %v, got %v\n", - objInfo.Size, objInfoCopy.Size) - } - - // CopyObject again but with wrong conditions - copyConds = CopyConditions{} - err = copyConds.SetUnmodified(time.Date(2014, time.April, 0, 0, 0, 0, 0, time.UTC)) - if err != nil { - t.Fatal("Error:", err) - } - err = copyConds.SetMatchETagExcept(objInfo.ETag) - if err != nil { - t.Fatal("Error:", err) - } - - // Perform the Copy which should fail - err = c.CopyObject(bucketName+"-copy", objectName+"-copy", copySource, copyConds) - if err == nil { - t.Fatal("Error:", err, bucketName+"-copy", objectName+"-copy should fail") - } - - // Remove all objects and buckets - err = c.RemoveObject(bucketName, objectName) - if err != nil { - t.Fatal("Error:", err) - } - - err = c.RemoveObject(bucketName+"-copy", objectName+"-copy") - if err != nil { - t.Fatal("Error:", err) - } - - err = c.RemoveBucket(bucketName) - if err != nil { - t.Fatal("Error:", err) - } - - err = c.RemoveBucket(bucketName + "-copy") - if err != nil { - t.Fatal("Error:", err) - } -} - -// TestEncryptionPutGet tests client side encryption -func TestEncryptionPutGet(t *testing.T) { - if testing.Short() { - t.Skip("skipping functional tests for the short runs") - } - - // Seed random based on current time. - rand.Seed(time.Now().Unix()) - - // Instantiate new minio client object. - c, err := New( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableSecurity)), - ) - if err != nil { - t.Fatal("Error:", err) - } - - // Enable tracing, write to stderr. - // c.TraceOn(os.Stderr) - - // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") - - // Generate a new random bucket name. - bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") - - // Make a new bucket. - err = c.MakeBucket(bucketName, "us-east-1") - if err != nil { - t.Fatal("Error:", err, bucketName) - } - - // Generate a symmetric key - symKey := encrypt.NewSymmetricKey([]byte("my-secret-key-00")) - - // Generate an assymmetric key from predefine public and private certificates - privateKey, err := hex.DecodeString( - "30820277020100300d06092a864886f70d0101010500048202613082025d" + - "0201000281810087b42ea73243a3576dc4c0b6fa245d339582dfdbddc20c" + - "bb8ab666385034d997210c54ba79275c51162a1221c3fb1a4c7c61131ca6" + - "5563b319d83474ef5e803fbfa7e52b889e1893b02586b724250de7ac6351" + - "cc0b7c638c980acec0a07020a78eed7eaa471eca4b92071394e061346c06" + - "15ccce2f465dee2080a89e43f29b5702030100010281801dd5770c3af8b3" + - "c85cd18cacad81a11bde1acfac3eac92b00866e142301fee565365aa9af4" + - "57baebf8bb7711054d071319a51dd6869aef3848ce477a0dc5f0dbc0c336" + - "5814b24c820491ae2bb3c707229a654427e03307fec683e6b27856688f08" + - "bdaa88054c5eeeb773793ff7543ee0fb0e2ad716856f2777f809ef7e6fa4" + - "41024100ca6b1edf89e8a8f93cce4b98c76c6990a09eb0d32ad9d3d04fbf" + - "0b026fa935c44f0a1c05dd96df192143b7bda8b110ec8ace28927181fd8c" + - "d2f17330b9b63535024100aba0260afb41489451baaeba423bee39bcbd1e" + - "f63dd44ee2d466d2453e683bf46d019a8baead3a2c7fca987988eb4d565e" + - "27d6be34605953f5034e4faeec9bdb0241009db2cb00b8be8c36710aff96" + - "6d77a6dec86419baca9d9e09a2b761ea69f7d82db2ae5b9aae4246599bb2" + - "d849684d5ab40e8802cfe4a2b358ad56f2b939561d2902404e0ead9ecafd" + - "bb33f22414fa13cbcc22a86bdf9c212ce1a01af894e3f76952f36d6c904c" + - "bd6a7e0de52550c9ddf31f1e8bfe5495f79e66a25fca5c20b3af5b870241" + - "0083456232aa58a8c45e5b110494599bda8dbe6a094683a0539ddd24e19d" + - "47684263bbe285ad953d725942d670b8f290d50c0bca3d1dc9688569f1d5" + - "9945cb5c7d") - - if err != nil { - t.Fatal(err) - } - - publicKey, err := hex.DecodeString("30819f300d06092a864886f70d010101050003818d003081890281810087" + - "b42ea73243a3576dc4c0b6fa245d339582dfdbddc20cbb8ab666385034d9" + - "97210c54ba79275c51162a1221c3fb1a4c7c61131ca65563b319d83474ef" + - "5e803fbfa7e52b889e1893b02586b724250de7ac6351cc0b7c638c980ace" + - "c0a07020a78eed7eaa471eca4b92071394e061346c0615ccce2f465dee20" + - "80a89e43f29b570203010001") - if err != nil { - t.Fatal(err) - } - - // Generate an asymmetric key - asymKey, err := encrypt.NewAsymmetricKey(privateKey, publicKey) - if err != nil { - t.Fatal(err) - } - - // Object custom metadata - customContentType := "custom/contenttype" - - testCases := []struct { - buf []byte - encKey encrypt.Key - }{ - {encKey: symKey, buf: bytes.Repeat([]byte("F"), 0)}, - {encKey: symKey, buf: bytes.Repeat([]byte("F"), 1)}, - {encKey: symKey, buf: bytes.Repeat([]byte("F"), 15)}, - {encKey: symKey, buf: bytes.Repeat([]byte("F"), 16)}, - {encKey: symKey, buf: bytes.Repeat([]byte("F"), 17)}, - {encKey: symKey, buf: bytes.Repeat([]byte("F"), 31)}, - {encKey: symKey, buf: bytes.Repeat([]byte("F"), 32)}, - {encKey: symKey, buf: bytes.Repeat([]byte("F"), 33)}, - {encKey: symKey, buf: bytes.Repeat([]byte("F"), 1024)}, - {encKey: symKey, buf: bytes.Repeat([]byte("F"), 1024*2)}, - {encKey: symKey, buf: bytes.Repeat([]byte("F"), 1024*1024)}, - - {encKey: asymKey, buf: bytes.Repeat([]byte("F"), 0)}, - {encKey: asymKey, buf: bytes.Repeat([]byte("F"), 1)}, - {encKey: asymKey, buf: bytes.Repeat([]byte("F"), 16)}, - {encKey: asymKey, buf: bytes.Repeat([]byte("F"), 32)}, - {encKey: asymKey, buf: bytes.Repeat([]byte("F"), 1024)}, - {encKey: asymKey, buf: bytes.Repeat([]byte("F"), 1024*1024)}, - } - - for i, testCase := range testCases { - // Generate a random object name - objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") - - // Secured object - cbcMaterials, err := encrypt.NewCBCSecureMaterials(testCase.encKey) - if err != nil { - t.Fatal(err) - } - - // Put encrypted data - _, err = c.PutEncryptedObject(bucketName, objectName, bytes.NewReader(testCase.buf), cbcMaterials, map[string][]string{"Content-Type": {customContentType}}, nil) - if err != nil { - t.Fatalf("Test %d, error: %v %v %v", i+1, err, bucketName, objectName) - } - - // Read the data back - r, err := c.GetEncryptedObject(bucketName, objectName, cbcMaterials) - if err != nil { - t.Fatalf("Test %d, error: %v %v %v", i+1, err, bucketName, objectName) - } - defer r.Close() - - // Compare the sent object with the received one - recvBuffer := bytes.NewBuffer([]byte{}) - if _, err = io.Copy(recvBuffer, r); err != nil { - t.Fatalf("Test %d, error: %v", i+1, err) - } - if recvBuffer.Len() != len(testCase.buf) { - t.Fatalf("Test %d, error: number of bytes of received object does not match, want %v, got %v\n", - i+1, len(testCase.buf), recvBuffer.Len()) - } - if !bytes.Equal(testCase.buf, recvBuffer.Bytes()) { - t.Fatalf("Test %d, error: Encrypted sent is not equal to decrypted, want `%x`, go `%x`", i+1, testCase.buf, recvBuffer.Bytes()) - } - - // Remove test object - err = c.RemoveObject(bucketName, objectName) - if err != nil { - t.Fatalf("Test %d, error: %v", i+1, err) - } - - } - - // Remove test bucket - err = c.RemoveBucket(bucketName) - if err != nil { - t.Fatal("Error:", err) - } - -} - -func TestBucketNotification(t *testing.T) { - if testing.Short() { - t.Skip("skipping functional tests for the short runs") - } - if os.Getenv("NOTIFY_BUCKET") == "" || - os.Getenv("NOTIFY_SERVICE") == "" || - os.Getenv("NOTIFY_REGION") == "" || - os.Getenv("NOTIFY_ACCOUNTID") == "" || - os.Getenv("NOTIFY_RESOURCE") == "" { - t.Skip("skipping notification test if not configured") - } - - // Seed random based on current time. - rand.Seed(time.Now().Unix()) - - c, err := New( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableSecurity)), - ) - if err != nil { - t.Fatal("Error:", err) - } - - // Enable to debug - // c.TraceOn(os.Stderr) - - // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") - - bucketName := os.Getenv("NOTIFY_BUCKET") - - topicArn := NewArn("aws", os.Getenv("NOTIFY_SERVICE"), os.Getenv("NOTIFY_REGION"), os.Getenv("NOTIFY_ACCOUNTID"), os.Getenv("NOTIFY_RESOURCE")) - queueArn := NewArn("aws", "dummy-service", "dummy-region", "dummy-accountid", "dummy-resource") - - topicConfig := NewNotificationConfig(topicArn) - topicConfig.AddEvents(ObjectCreatedAll, ObjectRemovedAll) - topicConfig.AddFilterSuffix("jpg") - - queueConfig := NewNotificationConfig(queueArn) - queueConfig.AddEvents(ObjectCreatedAll) - queueConfig.AddFilterPrefix("photos/") - - bNotification := BucketNotification{} - bNotification.AddTopic(topicConfig) - - // Add the same topicConfig again, should have no effect - // because it is duplicated - bNotification.AddTopic(topicConfig) - if len(bNotification.TopicConfigs) != 1 { - t.Fatal("Error: duplicated entry added") - } - - // Add and remove a queue config - bNotification.AddQueue(queueConfig) - bNotification.RemoveQueueByArn(queueArn) - - err = c.SetBucketNotification(bucketName, bNotification) - if err != nil { - t.Fatal("Error: ", err) - } - - bNotification, err = c.GetBucketNotification(bucketName) - if err != nil { - t.Fatal("Error: ", err) - } - - if len(bNotification.TopicConfigs) != 1 { - t.Fatal("Error: Topic config is empty") - } - - if bNotification.TopicConfigs[0].Filter.S3Key.FilterRules[0].Value != "jpg" { - t.Fatal("Error: cannot get the suffix") - } - - err = c.RemoveAllBucketNotification(bucketName) - if err != nil { - t.Fatal("Error: cannot delete bucket notification") - } -} - -// Tests comprehensive list of all methods. -func TestFunctional(t *testing.T) { - if testing.Short() { - t.Skip("skipping functional tests for the short runs") - } - - // Seed random based on current time. - rand.Seed(time.Now().Unix()) - - c, err := New( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableSecurity)), - ) - if err != nil { - t.Fatal("Error:", err) - } - - // Enable to debug - // c.TraceOn(os.Stderr) - - // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") - - // Generate a new random bucket name. - bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") - - // Make a new bucket. - err = c.MakeBucket(bucketName, "us-east-1") - if err != nil { - t.Fatal("Error:", err, bucketName) - } - - // Generate a random file name. - fileName := randString(60, rand.NewSource(time.Now().UnixNano()), "") - file, err := os.Create(fileName) - if err != nil { - t.Fatal("Error:", err) - } - for i := 0; i < 3; i++ { - buf := make([]byte, rand.Intn(1<<19)) - _, err = file.Write(buf) - if err != nil { - t.Fatal("Error:", err) - } - } - file.Close() - - // Verify if bucket exits and you have access. - var exists bool - exists, err = c.BucketExists(bucketName) - if err != nil { - t.Fatal("Error:", err, bucketName) - } - if !exists { - t.Fatal("Error: could not find ", bucketName) - } - - // Asserting the default bucket policy. - policyAccess, err := c.GetBucketPolicy(bucketName, "") - if err != nil { - t.Fatal("Error:", err) - } - if policyAccess != "none" { - t.Fatalf("Default bucket policy incorrect") - } - // Set the bucket policy to 'public readonly'. - err = c.SetBucketPolicy(bucketName, "", policy.BucketPolicyReadOnly) - if err != nil { - t.Fatal("Error:", err) - } - // should return policy `readonly`. - policyAccess, err = c.GetBucketPolicy(bucketName, "") - if err != nil { - t.Fatal("Error:", err) - } - if policyAccess != "readonly" { - t.Fatalf("Expected bucket policy to be readonly") - } - - // Make the bucket 'public writeonly'. - err = c.SetBucketPolicy(bucketName, "", policy.BucketPolicyWriteOnly) - if err != nil { - t.Fatal("Error:", err) - } - // should return policy `writeonly`. - policyAccess, err = c.GetBucketPolicy(bucketName, "") - if err != nil { - t.Fatal("Error:", err) - } - if policyAccess != "writeonly" { - t.Fatalf("Expected bucket policy to be writeonly") - } - // Make the bucket 'public read/write'. - err = c.SetBucketPolicy(bucketName, "", policy.BucketPolicyReadWrite) - if err != nil { - t.Fatal("Error:", err) - } - // should return policy `readwrite`. - policyAccess, err = c.GetBucketPolicy(bucketName, "") - if err != nil { - t.Fatal("Error:", err) - } - if policyAccess != "readwrite" { - t.Fatalf("Expected bucket policy to be readwrite") - } - // List all buckets. - buckets, err := c.ListBuckets() - if len(buckets) == 0 { - t.Fatal("Error: list buckets cannot be empty", buckets) - } - if err != nil { - t.Fatal("Error:", err) - } - - // Verify if previously created bucket is listed in list buckets. - bucketFound := false - for _, bucket := range buckets { - if bucket.Name == bucketName { - bucketFound = true - } - } - - // If bucket not found error out. - if !bucketFound { - t.Fatal("Error: bucket ", bucketName, "not found") - } - - objectName := bucketName + "unique" - - // Generate data - buf := bytes.Repeat([]byte("f"), 1<<19) - - n, err := c.PutObject(bucketName, objectName, bytes.NewReader(buf), "") - if err != nil { - t.Fatal("Error: ", err) - } - if n != int64(len(buf)) { - t.Fatal("Error: bad length ", n, len(buf)) - } - - n, err = c.PutObject(bucketName, objectName+"-nolength", bytes.NewReader(buf), "binary/octet-stream") - if err != nil { - t.Fatal("Error:", err, bucketName, objectName+"-nolength") - } - - if n != int64(len(buf)) { - t.Fatalf("Error: number of bytes does not match, want %v, got %v\n", len(buf), n) - } - - // Instantiate a done channel to close all listing. - doneCh := make(chan struct{}) - defer close(doneCh) - - objFound := false - isRecursive := true // Recursive is true. - for obj := range c.ListObjects(bucketName, objectName, isRecursive, doneCh) { - if obj.Key == objectName { - objFound = true - break - } - } - if !objFound { - t.Fatal("Error: object " + objectName + " not found.") - } - - objFound = false - isRecursive = true // Recursive is true. - for obj := range c.ListObjectsV2(bucketName, objectName, isRecursive, doneCh) { - if obj.Key == objectName { - objFound = true - break - } - } - if !objFound { - t.Fatal("Error: object " + objectName + " not found.") - } - - incompObjNotFound := true - for objIncompl := range c.ListIncompleteUploads(bucketName, objectName, isRecursive, doneCh) { - if objIncompl.Key != "" { - incompObjNotFound = false - break - } - } - if !incompObjNotFound { - t.Fatal("Error: unexpected dangling incomplete upload found.") - } - - newReader, err := c.GetObject(bucketName, objectName) - if err != nil { - t.Fatal("Error: ", err) - } - - newReadBytes, err := ioutil.ReadAll(newReader) - if err != nil { - t.Fatal("Error: ", err) - } - - if !bytes.Equal(newReadBytes, buf) { - t.Fatal("Error: bytes mismatch.") - } - - err = c.FGetObject(bucketName, objectName, fileName+"-f") - if err != nil { - t.Fatal("Error: ", err) - } - - // Generate presigned GET object url. - presignedGetURL, err := c.PresignedGetObject(bucketName, objectName, 3600*time.Second, nil) - if err != nil { - t.Fatal("Error: ", err) - } - - // Verify if presigned url works. - resp, err := http.Get(presignedGetURL.String()) - if err != nil { - t.Fatal("Error: ", err) - } - if resp.StatusCode != http.StatusOK { - t.Fatal("Error: ", resp.Status) - } - newPresignedBytes, err := ioutil.ReadAll(resp.Body) - if err != nil { - t.Fatal("Error: ", err) - } - if !bytes.Equal(newPresignedBytes, buf) { - t.Fatal("Error: bytes mismatch.") - } - - // Set request parameters. - reqParams := make(url.Values) - reqParams.Set("response-content-disposition", "attachment; filename=\"test.txt\"") - presignedGetURL, err = c.PresignedGetObject(bucketName, objectName, 3600*time.Second, reqParams) - if err != nil { - t.Fatal("Error: ", err) - } - // Verify if presigned url works. - resp, err = http.Get(presignedGetURL.String()) - if err != nil { - t.Fatal("Error: ", err) - } - if resp.StatusCode != http.StatusOK { - t.Fatal("Error: ", resp.Status) - } - newPresignedBytes, err = ioutil.ReadAll(resp.Body) - if err != nil { - t.Fatal("Error: ", err) - } - if !bytes.Equal(newPresignedBytes, buf) { - t.Fatal("Error: bytes mismatch for presigned GET URL.") - } - if resp.Header.Get("Content-Disposition") != "attachment; filename=\"test.txt\"" { - t.Fatalf("Error: wrong Content-Disposition received %s", resp.Header.Get("Content-Disposition")) - } - - presignedPutURL, err := c.PresignedPutObject(bucketName, objectName+"-presigned", 3600*time.Second) - if err != nil { - t.Fatal("Error: ", err) - } - - buf = bytes.Repeat([]byte("g"), 1<<19) - - req, err := http.NewRequest("PUT", presignedPutURL.String(), bytes.NewReader(buf)) - if err != nil { - t.Fatal("Error: ", err) - } - httpClient := &http.Client{ - // Setting a sensible time out of 30secs to wait for response - // headers. Request is pro-actively cancelled after 30secs - // with no response. - Timeout: 30 * time.Second, - Transport: http.DefaultTransport, - } - resp, err = httpClient.Do(req) - if err != nil { - t.Fatal("Error: ", err) - } - - newReader, err = c.GetObject(bucketName, objectName+"-presigned") - if err != nil { - t.Fatal("Error: ", err) - } - - newReadBytes, err = ioutil.ReadAll(newReader) - if err != nil { - t.Fatal("Error: ", err) - } - - if !bytes.Equal(newReadBytes, buf) { - t.Fatal("Error: bytes mismatch.") - } - - err = c.RemoveObject(bucketName, objectName) - if err != nil { - t.Fatal("Error: ", err) - } - err = c.RemoveObject(bucketName, objectName+"-f") - if err != nil { - t.Fatal("Error: ", err) - } - err = c.RemoveObject(bucketName, objectName+"-nolength") - if err != nil { - t.Fatal("Error: ", err) - } - err = c.RemoveObject(bucketName, objectName+"-presigned") - if err != nil { - t.Fatal("Error: ", err) - } - err = c.RemoveBucket(bucketName) - if err != nil { - t.Fatal("Error:", err) - } - err = c.RemoveBucket(bucketName) - if err == nil { - t.Fatal("Error:") - } - if err.Error() != "The specified bucket does not exist" { - t.Fatal("Error: ", err) - } - if err = os.Remove(fileName); err != nil { - t.Fatal("Error: ", err) - } - if err = os.Remove(fileName + "-f"); err != nil { - t.Fatal("Error: ", err) - } -} - -// Test for validating GetObject Reader* methods functioning when the -// object is modified in the object store. -func TestGetObjectObjectModified(t *testing.T) { - if testing.Short() { - t.Skip("skipping functional tests for the short runs") - } - - // Instantiate new minio client object. - c, err := NewV4( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableSecurity)), - ) - if err != nil { - t.Fatal("Error:", err) - } - - // Enable tracing, write to stderr. - // c.TraceOn(os.Stderr) - - // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") - - // Make a new bucket. - bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") - err = c.MakeBucket(bucketName, "us-east-1") - if err != nil { - t.Fatal("Error:", err, bucketName) - } - defer c.RemoveBucket(bucketName) - - // Upload an object. - objectName := "myobject" - content := "helloworld" - _, err = c.PutObject(bucketName, objectName, strings.NewReader(content), "application/text") - if err != nil { - t.Fatalf("Failed to upload %s/%s: %v", bucketName, objectName, err) - } - - defer c.RemoveObject(bucketName, objectName) - - reader, err := c.GetObject(bucketName, objectName) - if err != nil { - t.Fatalf("Failed to get object %s/%s: %v", bucketName, objectName, err) - } - defer reader.Close() - - // Read a few bytes of the object. - b := make([]byte, 5) - n, err := reader.ReadAt(b, 0) - if err != nil { - t.Fatalf("Failed to read object %s/%s at an offset: %v", bucketName, objectName, err) - } - - // Upload different contents to the same object while object is being read. - newContent := "goodbyeworld" - _, err = c.PutObject(bucketName, objectName, strings.NewReader(newContent), "application/text") - if err != nil { - t.Fatalf("Failed to upload %s/%s: %v", bucketName, objectName, err) - } - - // Confirm that a Stat() call in between doesn't change the Object's cached etag. - _, err = reader.Stat() - if err.Error() != s3ErrorResponseMap["PreconditionFailed"] { - t.Errorf("Expected Stat to fail with error %s but received %s", s3ErrorResponseMap["PreconditionFailed"], err.Error()) - } - - // Read again only to find object contents have been modified since last read. - _, err = reader.ReadAt(b, int64(n)) - if err.Error() != s3ErrorResponseMap["PreconditionFailed"] { - t.Errorf("Expected ReadAt to fail with error %s but received %s", s3ErrorResponseMap["PreconditionFailed"], err.Error()) - } -} - -// Test validates putObject to upload a file seeked at a given offset. -func TestPutObjectUploadSeekedObject(t *testing.T) { - if testing.Short() { - t.Skip("skipping functional tests for the short runs") - } - - // Instantiate new minio client object. - c, err := NewV4( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableSecurity)), - ) - if err != nil { - t.Fatal("Error:", err) - } - - // Enable tracing, write to stderr. - // c.TraceOn(os.Stderr) - - // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") - - // Make a new bucket. - bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") - err = c.MakeBucket(bucketName, "us-east-1") - if err != nil { - t.Fatal("Error:", err, bucketName) - } - defer c.RemoveBucket(bucketName) - - tempfile, err := ioutil.TempFile("", "minio-go-upload-test-") - if err != nil { - t.Fatal("Error:", err) - } - - var length = 120000 - data := bytes.Repeat([]byte("1"), length) - - if _, err = tempfile.Write(data); err != nil { - t.Fatal("Error:", err) - } - - objectName := fmt.Sprintf("test-file-%v", rand.Uint32()) - - offset := length / 2 - if _, err := tempfile.Seek(int64(offset), 0); err != nil { - t.Fatal("Error:", err) - } - - n, err := c.PutObject(bucketName, objectName, tempfile, "binary/octet-stream") - if err != nil { - t.Fatal("Error:", err) - } - if n != int64(length-offset) { - t.Fatalf("Invalid length returned, want %v, got %v", int64(length-offset), n) - } - tempfile.Close() - if err = os.Remove(tempfile.Name()); err != nil { - t.Fatal("Error:", err) - } - - length = int(n) - - obj, err := c.GetObject(bucketName, objectName) - if err != nil { - t.Fatal("Error:", err) - } - - n, err = obj.Seek(int64(offset), 0) - if err != nil { - t.Fatal("Error:", err) - } - if n != int64(offset) { - t.Fatalf("Invalid offset returned, want %v, got %v", int64(offset), n) - } - - n, err = c.PutObject(bucketName, objectName+"getobject", obj, "binary/octet-stream") - if err != nil { - t.Fatal("Error:", err) - } - if n != int64(length-offset) { - t.Fatalf("Invalid length returned, want %v, got %v", int64(length-offset), n) - } - - if err = c.RemoveObject(bucketName, objectName); err != nil { - t.Fatal("Error:", err) - } - - if err = c.RemoveObject(bucketName, objectName+"getobject"); err != nil { - t.Fatal("Error:", err) - } -} diff --git a/vendor/src/github.com/minio/minio-go/api_unit_test.go b/vendor/src/github.com/minio/minio-go/api_unit_test.go index 10d19aa10..2a9db3cb6 100644 --- a/vendor/src/github.com/minio/minio-go/api_unit_test.go +++ b/vendor/src/github.com/minio/minio-go/api_unit_test.go @@ -182,27 +182,6 @@ func TestValidBucketLocation(t *testing.T) { } } -// Tests temp file. -func TestTempFile(t *testing.T) { - tmpFile, err := newTempFile("testing") - if err != nil { - t.Fatal("Error:", err) - } - fileName := tmpFile.Name() - // Closing temporary file purges the file. - err = tmpFile.Close() - if err != nil { - t.Fatal("Error:", err) - } - st, err := os.Stat(fileName) - if err != nil && !os.IsNotExist(err) { - t.Fatal("Error:", err) - } - if err == nil && st != nil { - t.Fatal("Error: file should be deleted and should not exist.") - } -} - // Tests error response structure. func TestErrorResponse(t *testing.T) { var err error diff --git a/vendor/src/github.com/minio/minio-go/bucket-cache.go b/vendor/src/github.com/minio/minio-go/bucket-cache.go index 6d2a40f78..748fd01ee 100644 --- a/vendor/src/github.com/minio/minio-go/bucket-cache.go +++ b/vendor/src/github.com/minio/minio-go/bucket-cache.go @@ -213,20 +213,24 @@ func (c Client) getBucketLocationRequest(bucketName string) (*http.Request, erro signerType = credentials.SignatureAnonymous } - // Set sha256 sum for signature calculation only with signature version '4'. - switch { - case signerType.IsV4(): - var contentSha256 string - if c.secure { - contentSha256 = unsignedPayload - } else { - contentSha256 = hex.EncodeToString(sum256([]byte{})) - } - req.Header.Set("X-Amz-Content-Sha256", contentSha256) - req = s3signer.SignV4(*req, accessKeyID, secretAccessKey, sessionToken, "us-east-1") - case signerType.IsV2(): - req = s3signer.SignV2(*req, accessKeyID, secretAccessKey) + if signerType.IsAnonymous() { + return req, nil } + if signerType.IsV2() { + req = s3signer.SignV2(*req, accessKeyID, secretAccessKey) + return req, nil + } + + // Set sha256 sum for signature calculation only with signature version '4'. + var contentSha256 string + if c.secure { + contentSha256 = unsignedPayload + } else { + contentSha256 = hex.EncodeToString(sum256([]byte{})) + } + + req.Header.Set("X-Amz-Content-Sha256", contentSha256) + req = s3signer.SignV4(*req, accessKeyID, secretAccessKey, sessionToken, "us-east-1") return req, nil } diff --git a/vendor/src/github.com/minio/minio-go/constants.go b/vendor/src/github.com/minio/minio-go/constants.go index 6055bfdad..9771d2f92 100644 --- a/vendor/src/github.com/minio/minio-go/constants.go +++ b/vendor/src/github.com/minio/minio-go/constants.go @@ -18,10 +18,18 @@ package minio /// Multipart upload defaults. -// miniPartSize - minimum part size 64MiB per object after which +// absMinPartSize - absolute minimum part size (5 MiB) below which +// a part in a multipart upload may not be uploaded. +const absMinPartSize = 1024 * 1024 * 5 + +// minPartSize - minimum part size 64MiB per object after which // putObject behaves internally as multipart. const minPartSize = 1024 * 1024 * 64 +// copyPartSize - default (and maximum) part size to copy in a +// copy-object request (5GiB) +const copyPartSize = 1024 * 1024 * 1024 * 5 + // maxPartsCount - maximum number of parts for a single multipart session. const maxPartsCount = 10000 @@ -37,10 +45,6 @@ const maxSinglePutObjectSize = 1024 * 1024 * 1024 * 5 // Multipart operation. const maxMultipartPutObjectSize = 1024 * 1024 * 1024 * 1024 * 5 -// optimalReadBufferSize - optimal buffer 5MiB used for reading -// through Read operation. -const optimalReadBufferSize = 1024 * 1024 * 5 - // unsignedPayload - value to be set to X-Amz-Content-Sha256 header when // we don't want to sign the request payload const unsignedPayload = "UNSIGNED-PAYLOAD" diff --git a/vendor/src/github.com/minio/minio-go/copy-conditions.go b/vendor/src/github.com/minio/minio-go/copy-conditions.go deleted file mode 100644 index 65018aa09..000000000 --- a/vendor/src/github.com/minio/minio-go/copy-conditions.go +++ /dev/null @@ -1,99 +0,0 @@ -/* - * Minio Go Library for Amazon S3 Compatible Cloud Storage (C) 2016 Minio, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package minio - -import ( - "net/http" - "time" -) - -// copyCondition explanation: -// http://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectCOPY.html -// -// Example: -// -// copyCondition { -// key: "x-amz-copy-if-modified-since", -// value: "Tue, 15 Nov 1994 12:45:26 GMT", -// } -// -type copyCondition struct { - key string - value string -} - -// CopyConditions - copy conditions. -type CopyConditions struct { - conditions []copyCondition -} - -// NewCopyConditions - Instantiate new list of conditions. This -// function is left behind for backward compatibility. The idiomatic -// way to set an empty set of copy conditions is, -// ``copyConditions := CopyConditions{}``. -// -func NewCopyConditions() CopyConditions { - return CopyConditions{} -} - -// SetMatchETag - set match etag. -func (c *CopyConditions) SetMatchETag(etag string) error { - if etag == "" { - return ErrInvalidArgument("ETag cannot be empty.") - } - c.conditions = append(c.conditions, copyCondition{ - key: "x-amz-copy-source-if-match", - value: etag, - }) - return nil -} - -// SetMatchETagExcept - set match etag except. -func (c *CopyConditions) SetMatchETagExcept(etag string) error { - if etag == "" { - return ErrInvalidArgument("ETag cannot be empty.") - } - c.conditions = append(c.conditions, copyCondition{ - key: "x-amz-copy-source-if-none-match", - value: etag, - }) - return nil -} - -// SetUnmodified - set unmodified time since. -func (c *CopyConditions) SetUnmodified(modTime time.Time) error { - if modTime.IsZero() { - return ErrInvalidArgument("Modified since cannot be empty.") - } - c.conditions = append(c.conditions, copyCondition{ - key: "x-amz-copy-source-if-unmodified-since", - value: modTime.Format(http.TimeFormat), - }) - return nil -} - -// SetModified - set modified time since. -func (c *CopyConditions) SetModified(modTime time.Time) error { - if modTime.IsZero() { - return ErrInvalidArgument("Modified since cannot be empty.") - } - c.conditions = append(c.conditions, copyCondition{ - key: "x-amz-copy-source-if-modified-since", - value: modTime.Format(http.TimeFormat), - }) - return nil -} diff --git a/vendor/src/github.com/minio/minio-go/core.go b/vendor/src/github.com/minio/minio-go/core.go index be9388cec..4b1054a69 100644 --- a/vendor/src/github.com/minio/minio-go/core.go +++ b/vendor/src/github.com/minio/minio-go/core.go @@ -70,7 +70,13 @@ func (c Core) ListMultipartUploads(bucket, prefix, keyMarker, uploadIDMarker, de // PutObjectPart - Upload an object part. func (c Core) PutObjectPart(bucket, object, uploadID string, partID int, size int64, data io.Reader, md5Sum, sha256Sum []byte) (ObjectPart, error) { - return c.uploadPart(bucket, object, uploadID, data, partID, md5Sum, sha256Sum, size) + return c.PutObjectPartWithMetadata(bucket, object, uploadID, partID, size, data, md5Sum, sha256Sum, nil) +} + +// PutObjectPartWithMetadata - upload an object part with additional request metadata. +func (c Core) PutObjectPartWithMetadata(bucket, object, uploadID string, partID int, + size int64, data io.Reader, md5Sum, sha256Sum []byte, metadata map[string][]string) (ObjectPart, error) { + return c.uploadPart(bucket, object, uploadID, data, partID, md5Sum, sha256Sum, size, metadata) } // ListObjectParts - List uploaded parts of an incomplete upload.x @@ -80,7 +86,9 @@ func (c Core) ListObjectParts(bucket, object, uploadID string, partNumberMarker // CompleteMultipartUpload - Concatenate uploaded parts and commit to an object. func (c Core) CompleteMultipartUpload(bucket, object, uploadID string, parts []CompletePart) error { - _, err := c.completeMultipartUpload(bucket, object, uploadID, completeMultipartUpload{Parts: parts}) + _, err := c.completeMultipartUpload(bucket, object, uploadID, completeMultipartUpload{ + Parts: parts, + }) return err } diff --git a/vendor/src/github.com/minio/minio-go/core_test.go b/vendor/src/github.com/minio/minio-go/core_test.go index 81e1cd5bf..032e86ec5 100644 --- a/vendor/src/github.com/minio/minio-go/core_test.go +++ b/vendor/src/github.com/minio/minio-go/core_test.go @@ -18,14 +18,15 @@ package minio import ( "bytes" - "crypto/md5" - "io" - "math/rand" + "log" "os" "reflect" "testing" "time" + + "crypto/md5" + "math/rand" ) const ( @@ -35,6 +36,33 @@ const ( enableSecurity = "ENABLE_HTTPS" ) +// Minimum part size +const MinPartSize = 1024 * 1024 * 64 +const letterBytes = "abcdefghijklmnopqrstuvwxyz01234569" +const ( + letterIdxBits = 6 // 6 bits to represent a letter index + letterIdxMask = 1<= 0; { + if remain == 0 { + cache, remain = src.Int63(), letterIdxMax + } + if idx := int(cache & letterIdxMask); idx < len(letterBytes) { + b[i] = letterBytes[idx] + i-- + } + cache >>= letterIdxBits + remain-- + } + return prefix + string(b[0:30-len(prefix)]) +} + // Tests for Core GetObject() function. func TestGetObjectCore(t *testing.T) { if testing.Short() { @@ -209,6 +237,76 @@ func TestGetObjectCore(t *testing.T) { } } +// Tests GetObject to return Content-Encoding properly set +// and overrides any auto decoding. +func TestGetObjectContentEncoding(t *testing.T) { + if testing.Short() { + t.Skip("skipping functional tests for the short runs") + } + + // Seed random based on current time. + rand.Seed(time.Now().Unix()) + + // Instantiate new minio core client object. + c, err := NewCore( + os.Getenv(serverEndpoint), + os.Getenv(accessKey), + os.Getenv(secretKey), + mustParseBool(os.Getenv(enableSecurity)), + ) + if err != nil { + t.Fatal("Error:", err) + } + + // Enable tracing, write to stderr. + // c.TraceOn(os.Stderr) + + // Set user agent. + c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + + // Generate a new random bucket name. + bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") + + // Make a new bucket. + err = c.MakeBucket(bucketName, "us-east-1") + if err != nil { + t.Fatal("Error:", err, bucketName) + } + + // Generate data more than 32K + buf := bytes.Repeat([]byte("3"), rand.Intn(1<<20)+32*1024) + m := make(map[string][]string) + m["Content-Encoding"] = []string{"gzip"} + + // Save the data + objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") + n, err := c.Client.PutObjectWithMetadata(bucketName, objectName, bytes.NewReader(buf), m, nil) + if err != nil { + t.Fatal("Error:", err, bucketName, objectName) + } + + if n != int64(len(buf)) { + t.Fatalf("Error: number of bytes does not match, want %v, got %v\n", len(buf), n) + } + + reqHeaders := NewGetReqHeaders() + rwc, objInfo, err := c.GetObject(bucketName, objectName, reqHeaders) + if err != nil { + t.Fatalf("Error: %v", err) + } + rwc.Close() + if objInfo.Size <= 0 { + t.Fatalf("Unexpected size of the object %v, expected %v", objInfo.Size, n) + } + value, ok := objInfo.Metadata["Content-Encoding"] + if !ok { + t.Fatalf("Expected Content-Encoding metadata to be set.") + } + if value[0] != "gzip" { + t.Fatalf("Unexpected content-encoding found, want gzip, got %v", value) + } +} + // Tests get bucket policy core API. func TestGetBucketPolicy(t *testing.T) { if testing.Short() { @@ -373,3 +471,48 @@ func TestCorePutObject(t *testing.T) { t.Fatal("Error:", err) } } + +func TestCoreGetObjectMetadata(t *testing.T) { + if testing.Short() { + t.Skip("skipping functional tests for the short runs") + } + + core, err := NewCore( + os.Getenv(serverEndpoint), + os.Getenv(accessKey), + os.Getenv(secretKey), + mustParseBool(os.Getenv(enableSecurity))) + if err != nil { + log.Fatalln(err) + } + + // Generate a new random bucket name. + bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") + + // Make a new bucket. + err = core.MakeBucket(bucketName, "us-east-1") + if err != nil { + t.Fatal("Error:", err, bucketName) + } + + metadata := map[string][]string{ + "X-Amz-Meta-Key-1": {"Val-1"}, + } + + _, err = core.PutObject(bucketName, "my-objectname", 5, + bytes.NewReader([]byte("hello")), nil, nil, metadata) + if err != nil { + log.Fatalln(err) + } + + reader, objInfo, err := core.GetObject(bucketName, "my-objectname", + RequestHeaders{}) + if err != nil { + log.Fatalln(err) + } + defer reader.Close() + + if objInfo.Metadata.Get("X-Amz-Meta-Key-1") != "Val-1" { + log.Fatalln("Expected metadata to be available but wasn't") + } +} diff --git a/vendor/src/github.com/minio/minio-go/docs/API.md b/vendor/src/github.com/minio/minio-go/docs/API.md index fcdd91622..902a975d9 100644 --- a/vendor/src/github.com/minio/minio-go/docs/API.md +++ b/vendor/src/github.com/minio/minio-go/docs/API.md @@ -50,17 +50,21 @@ func main() { } ``` -| Bucket operations |Object operations | Encrypted Object operations | Presigned operations | Bucket Policy/Notification Operations | Client custom settings | -|:---|:---|:---|:---|:---|:---| -|[`MakeBucket`](#MakeBucket) |[`GetObject`](#GetObject) | [`NewSymmetricKey`](#NewSymmetricKey) | [`PresignedGetObject`](#PresignedGetObject) |[`SetBucketPolicy`](#SetBucketPolicy) | [`SetAppInfo`](#SetAppInfo) | -|[`ListBuckets`](#ListBuckets) |[`PutObject`](#PutObject) | [`NewAsymmetricKey`](#NewAsymmetricKey) |[`PresignedPutObject`](#PresignedPutObject) | [`GetBucketPolicy`](#GetBucketPolicy) | [`SetCustomTransport`](#SetCustomTransport) | -|[`BucketExists`](#BucketExists) |[`CopyObject`](#CopyObject) | [`GetEncryptedObject`](#GetEncryptedObject) |[`PresignedPostPolicy`](#PresignedPostPolicy) | [`ListBucketPolicies`](#ListBucketPolicies) | [`TraceOn`](#TraceOn) | -| [`RemoveBucket`](#RemoveBucket) |[`StatObject`](#StatObject) | [`PutObjectStreaming`](#PutObjectStreaming) | | [`SetBucketNotification`](#SetBucketNotification) | [`TraceOff`](#TraceOff) | -|[`ListObjects`](#ListObjects) |[`RemoveObject`](#RemoveObject) | [`PutEncryptedObject`](#PutEncryptedObject) | | [`GetBucketNotification`](#GetBucketNotification) | [`SetS3TransferAccelerate`](#SetS3TransferAccelerate) | -|[`ListObjectsV2`](#ListObjectsV2) | [`RemoveObjects`](#RemoveObjects) | | | [`RemoveAllBucketNotification`](#RemoveAllBucketNotification) | -|[`ListIncompleteUploads`](#ListIncompleteUploads) | [`RemoveIncompleteUpload`](#RemoveIncompleteUpload) | | | [`ListenBucketNotification`](#ListenBucketNotification) | -| | [`FPutObject`](#FPutObject) | | | | -| | [`FGetObject`](#FGetObject) | | | | +| Bucket operations | Object operations | Encrypted Object operations | Presigned operations | Bucket Policy/Notification Operations | Client custom settings | +| :--- | :--- | :--- | :--- | :--- | :--- | +| [`MakeBucket`](#MakeBucket) | [`GetObject`](#GetObject) | [`NewSymmetricKey`](#NewSymmetricKey) | [`PresignedGetObject`](#PresignedGetObject) | [`SetBucketPolicy`](#SetBucketPolicy) | [`SetAppInfo`](#SetAppInfo) | +| [`ListBuckets`](#ListBuckets) | [`PutObject`](#PutObject) | [`NewAsymmetricKey`](#NewAsymmetricKey) | [`PresignedPutObject`](#PresignedPutObject) | [`GetBucketPolicy`](#GetBucketPolicy) | [`SetCustomTransport`](#SetCustomTransport) | +| [`BucketExists`](#BucketExists) | [`CopyObject`](#CopyObject) | [`GetEncryptedObject`](#GetEncryptedObject) | [`PresignedPostPolicy`](#PresignedPostPolicy) | [`ListBucketPolicies`](#ListBucketPolicies) | [`TraceOn`](#TraceOn) | +| [`RemoveBucket`](#RemoveBucket) | [`StatObject`](#StatObject) | [`PutObjectStreaming`](#PutObjectStreaming) | | [`SetBucketNotification`](#SetBucketNotification) | [`TraceOff`](#TraceOff) | +| [`ListObjects`](#ListObjects) | [`RemoveObject`](#RemoveObject) | [`PutEncryptedObject`](#PutEncryptedObject) | | [`GetBucketNotification`](#GetBucketNotification) | [`SetS3TransferAccelerate`](#SetS3TransferAccelerate) | +| [`ListObjectsV2`](#ListObjectsV2) | [`RemoveObjects`](#RemoveObjects) | [`NewSSEInfo`](#NewSSEInfo) | | [`RemoveAllBucketNotification`](#RemoveAllBucketNotification) | | +| [`ListIncompleteUploads`](#ListIncompleteUploads) | [`RemoveIncompleteUpload`](#RemoveIncompleteUpload) | | | [`ListenBucketNotification`](#ListenBucketNotification) | | +| | [`FPutObject`](#FPutObject) | | | | | +| | [`FGetObject`](#FGetObject) | | | | | +| | [`ComposeObject`](#ComposeObject) | | | | | +| | [`NewSourceInfo`](#NewSourceInfo) | | | | | +| | [`NewDestinationInfo`](#NewDestinationInfo) | | | | | + ## 1. Constructor @@ -502,9 +506,11 @@ if err != nil { -### CopyObject(bucketName, objectName, objectSource string, conditions CopyConditions) error +### CopyObject(dst DestinationInfo, src SourceInfo) error -Copy a source object into a new object with the provided name in the provided bucket. +Create or replace an object through server-side copying of an existing object. It supports conditional copying, copying a part of an object and server-side encryption of destination and decryption of source. See the `SourceInfo` and `DestinationInfo` types for further details. + +To copy multiple source objects into a single destination object see the `ComposeObject` API. __Parameters__ @@ -512,50 +518,169 @@ __Parameters__ |Param |Type |Description | |:---|:---| :---| -|`bucketName` | _string_ |Name of the bucket | -|`objectName` | _string_ |Name of the object | -|`objectSource` | _string_ |Name of the source object | -|`conditions` | _CopyConditions_ |Collection of supported CopyObject conditions. [`x-amz-copy-source`, `x-amz-copy-source-if-match`, `x-amz-copy-source-if-none-match`, `x-amz-copy-source-if-unmodified-since`, `x-amz-copy-source-if-modified-since`]| +|`dst` | _DestinationInfo_ |Argument describing the destination object | +|`src` | _SourceInfo_ |Argument describing the source object | __Example__ ```go -// Use-case-1 -// To copy an existing object to a new object with _no_ copy conditions. -copyConds := minio.CopyConditions{} -err := minioClient.CopyObject("mybucket", "myobject", "my-sourcebucketname/my-sourceobjectname", copyConds) +// Use-case 1: Simple copy object with no conditions, etc +// Source object +src := minio.NewSourceInfo("my-sourcebucketname", "my-sourceobjectname", nil) + +// Destination object +dst, err := minio.NewDestinationInfo("my-bucketname", "my-objectname", nil, nil) if err != nil { fmt.Println(err) return } -// Use-case-2 -// To copy an existing object to a new object with the following copy conditions +// Copy object call +err = s3Client.CopyObject(dst, src) +if err != nil { + fmt.Println(err) + return +} + +// Use-case 2: Copy object with copy-conditions, and copying only part of the source object. // 1. that matches a given ETag // 2. and modified after 1st April 2014 // 3. but unmodified since 23rd April 2014 +// 4. copy only first 1MiB of object. -// Initialize empty copy conditions. -var copyConds = minio.CopyConditions{} +// Source object +src := minio.NewSourceInfo("my-sourcebucketname", "my-sourceobjectname", nil) -// copy object that matches the given ETag. -copyConds.SetMatchETag("31624deb84149d2f8ef9c385918b653a") +// Set matching ETag condition, copy object which matches the following ETag. +src.SetMatchETagCond("31624deb84149d2f8ef9c385918b653a") -// and modified after 1st April 2014 -copyConds.SetModified(time.Date(2014, time.April, 1, 0, 0, 0, 0, time.UTC)) +// Set modified condition, copy object modified since 2014 April 1. +src.SetModifiedSinceCond(time.Date(2014, time.April, 1, 0, 0, 0, 0, time.UTC)) -// but unmodified since 23rd April 2014 -copyConds.SetUnmodified(time.Date(2014, time.April, 23, 0, 0, 0, 0, time.UTC)) +// Set unmodified condition, copy object unmodified since 2014 April 23. +src.SetUnmodifiedSinceCond(time.Date(2014, time.April, 23, 0, 0, 0, 0, time.UTC)) -err := minioClient.CopyObject("mybucket", "myobject", "my-sourcebucketname/my-sourceobjectname", copyConds) +// Set copy-range of only first 1MiB of file. +src.SetRange(0, 1024*1024-1) + +// Destination object +dst, err := minio.NewDestinationInfo("my-bucketname", "my-objectname", nil, nil) +if err != nil { + fmt.Println(err) + return +} + +// Copy object call +err = s3Client.CopyObject(dst, src) if err != nil { fmt.Println(err) return } ``` + +### ComposeObject(dst DestinationInfo, srcs []SourceInfo) error + +Create an object by concatenating a list of source objects using +server-side copying. + +__Parameters__ + + +|Param |Type |Description | +|:---|:---|:---| +|`dst` | _minio.DestinationInfo_ |Struct with info about the object to be created. | +|`srcs` | _[]minio.SourceInfo_ |Slice of struct with info about source objects to be concatenated in order. | + + +__Example__ + + +```go +// Prepare source decryption key (here we assume same key to +// decrypt all source objects.) +decKey := minio.NewSSEInfo([]byte{1, 2, 3}, "") + +// Source objects to concatenate. We also specify decryption +// key for each +src1 := minio.NewSourceInfo("bucket1", "object1", decKey) +src1.SetMatchETag("31624deb84149d2f8ef9c385918b653a") + +src2 := minio.NewSourceInfo("bucket2", "object2", decKey) +src2.SetMatchETag("f8ef9c385918b653a31624deb84149d2") + +src3 := minio.NewSourceInfo("bucket3", "object3", decKey) +src3.SetMatchETag("5918b653a31624deb84149d2f8ef9c38") + +// Create slice of sources. +srcs := []minio.SourceInfo{src1, src2, src3} + +// Prepare destination encryption key +encKey := minio.NewSSEInfo([]byte{8, 9, 0}, "") + +// Create destination info +dst := minio.NewDestinationInfo("bucket", "object", encKey, nil) +err = s3Client.ComposeObject(dst, srcs) +if err != nil { + log.Println(err) + return +} + +log.Println("Composed object successfully.") +``` + + +### NewSourceInfo(bucket, object string, decryptSSEC *SSEInfo) SourceInfo + +Construct a `SourceInfo` object that can be used as the source for server-side copying operations like `CopyObject` and `ComposeObject`. This object can be used to set copy-conditions on the source. + +__Parameters__ + +| Param | Type | Description | +| :--- | :--- | :--- | +| `bucket` | _string_ | Name of the source bucket | +| `object` | _string_ | Name of the source object | +| `decryptSSEC` | _*minio.SSEInfo_ | Decryption info for the source object (`nil` without encryption) | + +__Example__ + +``` go +// No decryption parameter. +src := NewSourceInfo("bucket", "object", nil) + +// With decryption parameter. +decKey := NewSSEKey([]byte{1,2,3}, "") +src := NewSourceInfo("bucket", "object", decKey) +``` + + +### NewDestinationInfo(bucket, object string, encryptSSEC *SSEInfo, userMeta map[string]string) (DestinationInfo, error) + +Construct a `DestinationInfo` object that can be used as the destination object for server-side copying operations like `CopyObject` and `ComposeObject`. + +__Parameters__ + +| Param | Type | Description | +| :--- | :--- | :--- | +| `bucket` | _string_ | Name of the destination bucket | +| `object` | _string_ | Name of the destination object | +| `encryptSSEC` | _*minio.SSEInfo_ | Encryption info for the source object (`nil` without encryption) | +| `userMeta` | _map[string]string_ | User metadata to be set on the destination. If nil, with only one source, user-metadata is copied from source. | + +__Example__ + +``` go +// No encryption parameter. +dst, err := NewDestinationInfo("bucket", "object", nil, nil) + +// With encryption parameter. +encKey := NewSSEKey([]byte{1,2,3}, "") +dst, err := NewDecryptionInfo("bucket", "object", encKey, nil) +``` + + ### FPutObject(bucketName, objectName, filePath, contentType string) (length int64, err error) @@ -881,6 +1006,26 @@ if err != nil { } ``` + + +### NewSSEInfo(key []byte, algo string) SSEInfo + +Create a key object for use as encryption or decryption parameter in operations involving server-side-encryption with customer provided key (SSE-C). + +__Parameters__ + +| Param | Type | Description | +| :--- | :--- | :--- | +| `key` | _[]byte_ | Byte-slice of the raw, un-encoded binary key | +| `algo` | _string_ | Algorithm to use in encryption or decryption with the given key. Can be empty (defaults to `AES256`) | + +__Example__ + +``` go +// Key for use in encryption/decryption +keyInfo := NewSSEInfo([]byte{1,2,3}, "") +``` + ## 5. Presigned operations diff --git a/vendor/src/github.com/minio/minio-go/examples/s3/composeobject.go b/vendor/src/github.com/minio/minio-go/examples/s3/composeobject.go new file mode 100644 index 000000000..8aec6c158 --- /dev/null +++ b/vendor/src/github.com/minio/minio-go/examples/s3/composeobject.go @@ -0,0 +1,77 @@ +// +build ignore + +/* + * Minio Go Library for Amazon S3 Compatible Cloud Storage (C) 2016 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package main + +import ( + "log" + + minio "github.com/minio/minio-go" +) + +func main() { + // Note: YOUR-ACCESSKEYID, YOUR-SECRETACCESSKEY, my-testfile, my-bucketname and + // my-objectname are dummy values, please replace them with original values. + + // Requests are always secure (HTTPS) by default. Set secure=false to enable insecure (HTTP) access. + // This boolean value is the last argument for New(). + + // New returns an Amazon S3 compatible client object. API compatibility (v2 or v4) is automatically + // determined based on the Endpoint value. + s3Client, err := minio.New("s3.amazonaws.com", "YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", true) + if err != nil { + log.Fatalln(err) + } + + // Enable trace. + // s3Client.TraceOn(os.Stderr) + + // Prepare source decryption key (here we assume same key to + // decrypt all source objects.) + decKey := minio.NewSSEInfo([]byte{1, 2, 3}, "") + + // Source objects to concatenate. We also specify decryption + // key for each + src1 := minio.NewSourceInfo("bucket1", "object1", &decKey) + src1.SetMatchETagCond("31624deb84149d2f8ef9c385918b653a") + + src2 := minio.NewSourceInfo("bucket2", "object2", &decKey) + src2.SetMatchETagCond("f8ef9c385918b653a31624deb84149d2") + + src3 := minio.NewSourceInfo("bucket3", "object3", &decKey) + src3.SetMatchETagCond("5918b653a31624deb84149d2f8ef9c38") + + // Create slice of sources. + srcs := []minio.SourceInfo{src1, src2, src3} + + // Prepare destination encryption key + encKey := minio.NewSSEInfo([]byte{8, 9, 0}, "") + + // Create destination info + dst, err := minio.NewDestinationInfo("bucket", "object", &encKey, nil) + if err != nil { + log.Fatalln(err) + } + + err = s3Client.ComposeObject(dst, srcs) + if err != nil { + log.Fatalln(err) + } + + log.Println("Composed object successfully.") +} diff --git a/vendor/src/github.com/minio/minio-go/examples/s3/copyobject.go b/vendor/src/github.com/minio/minio-go/examples/s3/copyobject.go index a9ec78fee..c1d92d73a 100644 --- a/vendor/src/github.com/minio/minio-go/examples/s3/copyobject.go +++ b/vendor/src/github.com/minio/minio-go/examples/s3/copyobject.go @@ -42,24 +42,31 @@ func main() { // Enable trace. // s3Client.TraceOn(os.Stderr) + // Source object + src := minio.NewSourceInfo("my-sourcebucketname", "my-sourceobjectname", nil) + // All following conditions are allowed and can be combined together. - // Set copy conditions. - var copyConds = minio.CopyConditions{} // Set modified condition, copy object modified since 2014 April. - copyConds.SetModified(time.Date(2014, time.April, 0, 0, 0, 0, 0, time.UTC)) + src.SetModifiedSinceCond(time.Date(2014, time.April, 0, 0, 0, 0, 0, time.UTC)) // Set unmodified condition, copy object unmodified since 2014 April. - // copyConds.SetUnmodified(time.Date(2014, time.April, 0, 0, 0, 0, 0, time.UTC)) + // src.SetUnmodifiedSinceCond(time.Date(2014, time.April, 0, 0, 0, 0, 0, time.UTC)) // Set matching ETag condition, copy object which matches the following ETag. - // copyConds.SetMatchETag("31624deb84149d2f8ef9c385918b653a") + // src.SetMatchETagCond("31624deb84149d2f8ef9c385918b653a") // Set matching ETag except condition, copy object which does not match the following ETag. - // copyConds.SetMatchETagExcept("31624deb84149d2f8ef9c385918b653a") + // src.SetMatchETagExceptCond("31624deb84149d2f8ef9c385918b653a") + + // Destination object + dst, err := minio.NewDestinationInfo("my-bucketname", "my-objectname", nil, nil) + if err != nil { + log.Fatalln(err) + } // Initiate copy object. - err = s3Client.CopyObject("my-bucketname", "my-objectname", "/my-sourcebucketname/my-sourceobjectname", copyConds) + err = s3Client.CopyObject(dst, src) if err != nil { log.Fatalln(err) } diff --git a/vendor/src/github.com/minio/minio-go/examples/s3/putobject-progress.go b/vendor/src/github.com/minio/minio-go/examples/s3/putobject-progress.go index 1179fd787..26e77b9e6 100644 --- a/vendor/src/github.com/minio/minio-go/examples/s3/putobject-progress.go +++ b/vendor/src/github.com/minio/minio-go/examples/s3/putobject-progress.go @@ -55,7 +55,9 @@ func main() { progress := pb.New64(objectInfo.Size) progress.Start() - n, err := s3Client.PutObjectWithProgress("my-bucketname", "my-objectname-progress", reader, "application/octet-stream", progress) + n, err := s3Client.PutObjectWithProgress("my-bucketname", "my-objectname-progress", reader, map[string][]string{ + "Content-Type": []string{"application/octet-stream"}, + }, progress) if err != nil { log.Fatalln(err) } diff --git a/vendor/src/github.com/minio/minio-go/examples/s3/putobject-s3-accelerate.go b/vendor/src/github.com/minio/minio-go/examples/s3/putobject-s3-accelerate.go index e47976f2e..a26415c7a 100644 --- a/vendor/src/github.com/minio/minio-go/examples/s3/putobject-s3-accelerate.go +++ b/vendor/src/github.com/minio/minio-go/examples/s3/putobject-s3-accelerate.go @@ -40,7 +40,7 @@ func main() { } // Enable S3 transfer accelerate endpoint. - s3Client.S3TransferAccelerate("s3-accelerate.amazonaws.com") + s3Client.SetS3TransferAccelerate("s3-accelerate.amazonaws.com") object, err := os.Open("my-testfile") if err != nil { diff --git a/vendor/src/github.com/minio/minio-go/functional_tests.go b/vendor/src/github.com/minio/minio-go/functional_tests.go new file mode 100644 index 000000000..991e2d3c0 --- /dev/null +++ b/vendor/src/github.com/minio/minio-go/functional_tests.go @@ -0,0 +1,4058 @@ +// +build ignore + +/* + * Minio Go Library for Amazon S3 Compatible Cloud Storage (C) 2015 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package main + +import ( + "bytes" + "encoding/hex" + "errors" + "fmt" + "io" + "io/ioutil" + "math/rand" + "net/http" + "net/url" + "os" + "path" + "reflect" + "runtime" + "strconv" + "strings" + "time" + + minio "github.com/minio/minio-go" + log "github.com/sirupsen/logrus" + logrus "github.com/sirupsen/logrus" + + "github.com/minio/minio-go/pkg/encrypt" + "github.com/minio/minio-go/pkg/policy" +) + +// MinPartSize ... Minimum part size +const MinPartSize = 1024 * 1024 * 64 +const letterBytes = "abcdefghijklmnopqrstuvwxyz01234569" +const ( + letterIdxBits = 6 // 6 bits to represent a letter index + letterIdxMask = 1<= 0; { + if remain == 0 { + cache, remain = src.Int63(), letterIdxMax + } + if idx := int(cache & letterIdxMask); idx < len(letterBytes) { + b[i] = letterBytes[idx] + i-- + } + cache >>= letterIdxBits + remain-- + } + return prefix + string(b[0:30-len(prefix)]) +} + +func isQuickMode() bool { + return os.Getenv("MODE") == "quick" +} + +// Tests bucket re-create errors. +func testMakeBucketError() { + logger().Info() + + if os.Getenv(serverEndpoint) != "s3.amazonaws.com" { + logger().Info("skipping region functional tests for non s3 runs") + return + } + + // Seed random based on current time. + rand.Seed(time.Now().Unix()) + + // Instantiate new minio client object. + c, err := minio.New( + os.Getenv(serverEndpoint), + os.Getenv(accessKey), + os.Getenv(secretKey), + mustParseBool(os.Getenv(enableHTTPS)), + ) + if err != nil { + logger().Fatalf("Error: %s", err) + } + + // Enable tracing, write to stderr. + // c.TraceOn(os.Stderr) + + // Set user agent. + c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + + // Generate a new random bucket name. + bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") + + // Make a new bucket in 'eu-central-1'. + if err = c.MakeBucket(bucketName, "eu-central-1"); err != nil { + logger().Fatal("Error:", err, bucketName) + } + if err = c.MakeBucket(bucketName, "eu-central-1"); err == nil { + logger().Fatal("Error: make bucket should should fail for", bucketName) + } + // Verify valid error response from server. + if minio.ToErrorResponse(err).Code != "BucketAlreadyExists" && + minio.ToErrorResponse(err).Code != "BucketAlreadyOwnedByYou" { + logger().Fatal("Error: Invalid error returned by server", err) + } + if err = c.RemoveBucket(bucketName); err != nil { + logger().Fatal("Error:", err, bucketName) + } +} + +// Tests various bucket supported formats. +func testMakeBucketRegions() { + logger().Info() + + if os.Getenv(serverEndpoint) != "s3.amazonaws.com" { + logger().Info("skipping region functional tests for non s3 runs") + return + } + + // Seed random based on current time. + rand.Seed(time.Now().Unix()) + + // Instantiate new minio client object. + c, err := minio.New( + os.Getenv(serverEndpoint), + os.Getenv(accessKey), + os.Getenv(secretKey), + mustParseBool(os.Getenv(enableHTTPS)), + ) + if err != nil { + logger().Fatal("Error:", err) + } + + // Enable tracing, write to stderr. + // c.TraceOn(os.Stderr) + + // Set user agent. + c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + + // Generate a new random bucket name. + bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") + + // Make a new bucket in 'eu-central-1'. + if err = c.MakeBucket(bucketName, "eu-central-1"); err != nil { + logger().Fatal("Error:", err, bucketName) + } + + if err = c.RemoveBucket(bucketName); err != nil { + logger().Fatal("Error:", err, bucketName) + } + + // Make a new bucket with '.' in its name, in 'us-west-2'. This + // request is internally staged into a path style instead of + // virtual host style. + if err = c.MakeBucket(bucketName+".withperiod", "us-west-2"); err != nil { + logger().Fatal("Error:", err, bucketName+".withperiod") + } + + // Remove the newly created bucket. + if err = c.RemoveBucket(bucketName + ".withperiod"); err != nil { + logger().Fatal("Error:", err, bucketName+".withperiod") + } +} + +// Test PutObject using a large data to trigger multipart readat +func testPutObjectReadAt() { + logger().Info() + + // Seed random based on current time. + rand.Seed(time.Now().Unix()) + + // Instantiate new minio client object. + c, err := minio.New( + os.Getenv(serverEndpoint), + os.Getenv(accessKey), + os.Getenv(secretKey), + mustParseBool(os.Getenv(enableHTTPS)), + ) + if err != nil { + logger().Fatal("Error:", err) + } + + // Enable tracing, write to stderr. + // c.TraceOn(os.Stderr) + + // Set user agent. + c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + + // Generate a new random bucket name. + bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") + + // Make a new bucket. + err = c.MakeBucket(bucketName, "us-east-1") + if err != nil { + logger().Fatal("Error:", err, bucketName) + } + + // Generate data using 4 parts so that all 3 'workers' are utilized and a part is leftover. + // Use different data for each part for multipart tests to ensure part order at the end. + var buf = getDataBuffer("datafile-65-MB", MinPartSize) + + // Save the data + objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") + // Object content type + objectContentType := "binary/octet-stream" + + n, err := c.PutObject(bucketName, objectName, bytes.NewReader(buf), objectContentType) + if err != nil { + logger().Fatal("Error:", err, bucketName, objectName) + } + + if n != int64(len(buf)) { + logger().Fatalf("Error: number of bytes does not match, want %v, got %v\n", len(buf), n) + } + + // Read the data back + r, err := c.GetObject(bucketName, objectName) + if err != nil { + logger().Fatal("Error:", err, bucketName, objectName) + } + + st, err := r.Stat() + if err != nil { + logger().Fatal("Error:", err, bucketName, objectName) + } + if st.Size != int64(len(buf)) { + logger().Fatalf("Error: number of bytes in stat does not match, want %v, got %v\n", + len(buf), st.Size) + } + if st.ContentType != objectContentType { + logger().Fatalf("Error: Content types don't match, expected: %+v, found: %+v\n", objectContentType, st.ContentType) + } + if err := r.Close(); err != nil { + logger().Fatal("Error:", err) + } + if err := r.Close(); err == nil { + logger().Fatal("Error: object is already closed, should return error") + } + + err = c.RemoveObject(bucketName, objectName) + if err != nil { + logger().Fatal("Error: ", err) + } + err = c.RemoveBucket(bucketName) + if err != nil { + logger().Fatal("Error:", err) + } +} + +// Test PutObject using a large data to trigger multipart readat +func testPutObjectWithMetadata() { + logger().Info() + if isQuickMode() { + logger().Info("skipping functional tests for short runs") + return + } + + // Seed random based on current time. + rand.Seed(time.Now().Unix()) + + // Instantiate new minio client object. + c, err := minio.New( + os.Getenv(serverEndpoint), + os.Getenv(accessKey), + os.Getenv(secretKey), + mustParseBool(os.Getenv(enableHTTPS)), + ) + if err != nil { + logger().Fatal("Error:", err) + } + + // Enable tracing, write to stderr. + // c.TraceOn(os.Stderr) + + // Set user agent. + c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + + // Generate a new random bucket name. + bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") + + // Make a new bucket. + err = c.MakeBucket(bucketName, "us-east-1") + if err != nil { + logger().Fatal("Error:", err, bucketName) + } + + // Generate data using 2 parts + // Use different data in each part for multipart tests to ensure part order at the end. + var buf = getDataBuffer("datafile-65-MB", MinPartSize) + + // Save the data + objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") + + // Object custom metadata + customContentType := "custom/contenttype" + + n, err := c.PutObjectWithMetadata(bucketName, objectName, bytes.NewReader(buf), map[string][]string{ + "Content-Type": {customContentType}, + }, nil) + if err != nil { + logger().Fatal("Error:", err, bucketName, objectName) + } + + if n != int64(len(buf)) { + logger().Fatalf("Error: number of bytes does not match, want %v, got %v\n", len(buf), n) + } + + // Read the data back + r, err := c.GetObject(bucketName, objectName) + if err != nil { + logger().Fatal("Error:", err, bucketName, objectName) + } + + st, err := r.Stat() + if err != nil { + logger().Fatal("Error:", err, bucketName, objectName) + } + if st.Size != int64(len(buf)) { + logger().Fatalf("Error: number of bytes in stat does not match, want %v, got %v\n", + len(buf), st.Size) + } + if st.ContentType != customContentType { + logger().Fatalf("Error: Expected and found content types do not match, want %v, got %v\n", + customContentType, st.ContentType) + } + if err := r.Close(); err != nil { + logger().Fatal("Error:", err) + } + if err := r.Close(); err == nil { + logger().Fatal("Error: object is already closed, should return error") + } + + err = c.RemoveObject(bucketName, objectName) + if err != nil { + logger().Fatal("Error: ", err) + } + err = c.RemoveBucket(bucketName) + if err != nil { + logger().Fatal("Error:", err) + } +} + +// Test put object with streaming signature. +func testPutObjectStreaming() { + logger().Info() + + // Seed random based on current time. + rand.Seed(time.Now().Unix()) + + // Instantiate new minio client object. + c, err := minio.NewV4( + os.Getenv(serverEndpoint), + os.Getenv(accessKey), + os.Getenv(secretKey), + mustParseBool(os.Getenv(enableHTTPS)), + ) + if err != nil { + logger().Fatal("Error:", err) + } + + // Enable tracing, write to stderr. + // c.TraceOn(os.Stderr) + + // Set user agent. + c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + + // Generate a new random bucket name. + bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), + "minio-go-test") + + // Make a new bucket. + err = c.MakeBucket(bucketName, "us-east-1") + if err != nil { + logger().Fatal("Error:", err, bucketName) + } + + // Upload an object. + sizes := []int64{0, 64*1024 - 1, 64 * 1024} + objectName := "test-object" + for i, size := range sizes { + data := bytes.Repeat([]byte("a"), int(size)) + n, err := c.PutObjectStreaming(bucketName, objectName, bytes.NewReader(data)) + if err != nil { + logger().Fatalf("Test %d Error: %v %s %s", i+1, err, bucketName, objectName) + } + + if n != size { + log.Error(fmt.Errorf("Test %d Expected upload object size %d but got %d", i+1, size, n)) + } + } + + // Remove the object. + err = c.RemoveObject(bucketName, objectName) + if err != nil { + logger().Fatal("Error:", err) + } + + // Remove the bucket. + err = c.RemoveBucket(bucketName) + if err != nil { + logger().Fatal("Error:", err) + } +} + +// Test listing partially uploaded objects. +func testListPartiallyUploaded() { + logger().Info() + + // Seed random based on current time. + rand.Seed(time.Now().Unix()) + + // Instantiate new minio client object. + c, err := minio.New( + os.Getenv(serverEndpoint), + os.Getenv(accessKey), + os.Getenv(secretKey), + mustParseBool(os.Getenv(enableHTTPS)), + ) + if err != nil { + logger().Fatal("Error:", err) + } + + // Set user agent. + c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + + // Enable tracing, write to stdout. + // c.TraceOn(os.Stderr) + + // Generate a new random bucket name. + bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") + + // Make a new bucket. + err = c.MakeBucket(bucketName, "us-east-1") + if err != nil { + logger().Fatal("Error:", err, bucketName) + } + + r := bytes.NewReader(bytes.Repeat([]byte("0"), MinPartSize*2)) + + reader, writer := io.Pipe() + go func() { + i := 0 + for i < 25 { + _, cerr := io.CopyN(writer, r, (MinPartSize*2)/25) + if cerr != nil { + logger().Fatal("Error:", cerr, bucketName) + } + i++ + r.Seek(0, 0) + } + writer.CloseWithError(errors.New("proactively closed to be verified later")) + }() + + objectName := bucketName + "-resumable" + _, err = c.PutObject(bucketName, objectName, reader, "application/octet-stream") + if err == nil { + logger().Fatal("Error: PutObject should fail.") + } + if !strings.Contains(err.Error(), "proactively closed to be verified later") { + logger().Fatal("Error:", err) + } + + doneCh := make(chan struct{}) + defer close(doneCh) + isRecursive := true + multiPartObjectCh := c.ListIncompleteUploads(bucketName, objectName, isRecursive, doneCh) + for multiPartObject := range multiPartObjectCh { + if multiPartObject.Err != nil { + logger().Fatalf("Error: Error when listing incomplete upload") + } + } + + err = c.RemoveBucket(bucketName) + if err != nil { + logger().Fatal("Error:", err) + } +} + +// Test get object seeker from the end, using whence set to '2'. +func testGetObjectSeekEnd() { + logger().Info() + + // Seed random based on current time. + rand.Seed(time.Now().Unix()) + + // Instantiate new minio client object. + c, err := minio.New( + os.Getenv(serverEndpoint), + os.Getenv(accessKey), + os.Getenv(secretKey), + mustParseBool(os.Getenv(enableHTTPS)), + ) + if err != nil { + logger().Fatal("Error:", err) + } + + // Enable tracing, write to stderr. + // c.TraceOn(os.Stderr) + + // Set user agent. + c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + + // Generate a new random bucket name. + bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") + + // Make a new bucket. + err = c.MakeBucket(bucketName, "us-east-1") + if err != nil { + logger().Fatal("Error:", err, bucketName) + } + var buf = getDataBuffer("datafile-33-kB", rand.Intn(1<<20)+32*1024) + + // Save the data + objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") + n, err := c.PutObject(bucketName, objectName, bytes.NewReader(buf), "binary/octet-stream") + if err != nil { + logger().Fatal("Error:", err, bucketName, objectName) + } + + if n != int64(len(buf)) { + logger().Fatalf("Error: number of bytes does not match, want %v, got %v\n", len(buf), n) + } + + // Read the data back + r, err := c.GetObject(bucketName, objectName) + if err != nil { + logger().Fatal("Error:", err, bucketName, objectName) + } + + st, err := r.Stat() + if err != nil { + logger().Fatal("Error:", err, bucketName, objectName) + } + if st.Size != int64(len(buf)) { + logger().Fatalf("Error: number of bytes in stat does not match, want %v, got %v\n", + len(buf), st.Size) + } + + pos, err := r.Seek(-100, 2) + if err != nil { + logger().Fatal("Error:", err, bucketName, objectName) + } + if pos != st.Size-100 { + logger().Fatalf("Expected %d, got %d instead", pos, st.Size-100) + } + buf2 := make([]byte, 100) + m, err := io.ReadFull(r, buf2) + if err != nil { + logger().Fatal("Error: reading through io.ReadFull", err, bucketName, objectName) + } + if m != len(buf2) { + logger().Fatalf("Expected %d bytes, got %d", len(buf2), m) + } + hexBuf1 := fmt.Sprintf("%02x", buf[len(buf)-100:]) + hexBuf2 := fmt.Sprintf("%02x", buf2[:m]) + if hexBuf1 != hexBuf2 { + logger().Fatalf("Expected %s, got %s instead", hexBuf1, hexBuf2) + } + pos, err = r.Seek(-100, 2) + if err != nil { + logger().Fatal("Error:", err, bucketName, objectName) + } + if pos != st.Size-100 { + logger().Fatalf("Expected %d, got %d instead", pos, st.Size-100) + } + if err = r.Close(); err != nil { + logger().Fatal("Error:", err, bucketName, objectName) + } +} + +// Test get object reader to not throw error on being closed twice. +func testGetObjectClosedTwice() { + logger().Info() + + // Seed random based on current time. + rand.Seed(time.Now().Unix()) + + // Instantiate new minio client object. + c, err := minio.New( + os.Getenv(serverEndpoint), + os.Getenv(accessKey), + os.Getenv(secretKey), + mustParseBool(os.Getenv(enableHTTPS)), + ) + if err != nil { + logger().Fatal("Error:", err) + } + + // Enable tracing, write to stderr. + // c.TraceOn(os.Stderr) + + // Set user agent. + c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + + // Generate a new random bucket name. + bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") + + // Make a new bucket. + err = c.MakeBucket(bucketName, "us-east-1") + if err != nil { + logger().Fatal("Error:", err, bucketName) + } + + // Generate data more than 32K + var buf = getDataBuffer("datafile-33-kB", rand.Intn(1<<20)+32*1024) + // Save the data + objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") + n, err := c.PutObject(bucketName, objectName, bytes.NewReader(buf), "binary/octet-stream") + if err != nil { + logger().Fatal("Error:", err, bucketName, objectName) + } + + if n != int64(len(buf)) { + logger().Fatalf("Error: number of bytes does not match, want %v, got %v\n", len(buf), n) + } + + // Read the data back + r, err := c.GetObject(bucketName, objectName) + if err != nil { + logger().Fatal("Error:", err, bucketName, objectName) + } + + st, err := r.Stat() + if err != nil { + logger().Fatal("Error:", err, bucketName, objectName) + } + if st.Size != int64(len(buf)) { + logger().Fatalf("Error: number of bytes in stat does not match, want %v, got %v\n", + len(buf), st.Size) + } + if err := r.Close(); err != nil { + logger().Fatal("Error:", err) + } + if err := r.Close(); err == nil { + logger().Fatal("Error: object is already closed, should return error") + } + + err = c.RemoveObject(bucketName, objectName) + if err != nil { + logger().Fatal("Error: ", err) + } + err = c.RemoveBucket(bucketName) + if err != nil { + logger().Fatal("Error:", err) + } +} + +// Test removing multiple objects with Remove API +func testRemoveMultipleObjects() { + logger().Info() + // Seed random based on current time. + rand.Seed(time.Now().Unix()) + + // Instantiate new minio client object. + c, err := minio.New( + os.Getenv(serverEndpoint), + os.Getenv(accessKey), + os.Getenv(secretKey), + mustParseBool(os.Getenv(enableHTTPS)), + ) + + if err != nil { + logger().Fatal("Error:", err) + } + + // Set user agent. + c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + + // Enable tracing, write to stdout. + // c.TraceOn(os.Stderr) + + // Generate a new random bucket name. + bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") + + // Make a new bucket. + err = c.MakeBucket(bucketName, "us-east-1") + if err != nil { + logger().Fatal("Error:", err, bucketName) + } + + r := bytes.NewReader(bytes.Repeat([]byte("a"), 8)) + + // Multi remove of 1100 objects + nrObjects := 1100 + + objectsCh := make(chan string) + + go func() { + defer close(objectsCh) + // Upload objects and send them to objectsCh + for i := 0; i < nrObjects; i++ { + objectName := "sample" + strconv.Itoa(i) + ".txt" + _, err = c.PutObject(bucketName, objectName, r, "application/octet-stream") + if err != nil { + log.Error("Error: PutObject shouldn't fail.", err) + continue + } + objectsCh <- objectName + } + }() + + // Call RemoveObjects API + errorCh := c.RemoveObjects(bucketName, objectsCh) + + // Check if errorCh doesn't receive any error + select { + case r, more := <-errorCh: + if more { + logger().Fatalf("Unexpected error, objName(%v) err(%v)", r.ObjectName, r.Err) + } + } + + // Clean the bucket created by the test + err = c.RemoveBucket(bucketName) + if err != nil { + logger().Fatal("Error:", err) + } +} + +// Tests removing partially uploaded objects. +func testRemovePartiallyUploaded() { + logger().Info() + + // Seed random based on current time. + rand.Seed(time.Now().Unix()) + + // Instantiate new minio client object. + c, err := minio.New( + os.Getenv(serverEndpoint), + os.Getenv(accessKey), + os.Getenv(secretKey), + mustParseBool(os.Getenv(enableHTTPS)), + ) + if err != nil { + logger().Fatal("Error:", err) + } + + // Set user agent. + c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + + // Enable tracing, write to stdout. + // c.TraceOn(os.Stderr) + + // Generate a new random bucket name. + bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") + + // Make a new bucket. + err = c.MakeBucket(bucketName, "us-east-1") + if err != nil { + logger().Fatal("Error:", err, bucketName) + } + + r := bytes.NewReader(bytes.Repeat([]byte("a"), 128*1024)) + + reader, writer := io.Pipe() + go func() { + i := 0 + for i < 25 { + _, cerr := io.CopyN(writer, r, 128*1024) + if cerr != nil { + logger().Fatal("Error:", cerr, bucketName) + } + i++ + r.Seek(0, 0) + } + writer.CloseWithError(errors.New("proactively closed to be verified later")) + }() + + objectName := bucketName + "-resumable" + _, err = c.PutObject(bucketName, objectName, reader, "application/octet-stream") + if err == nil { + logger().Fatal("Error: PutObject should fail.") + } + if !strings.Contains(err.Error(), "proactively closed to be verified later") { + logger().Fatal("Error:", err) + } + err = c.RemoveIncompleteUpload(bucketName, objectName) + if err != nil { + logger().Fatal("Error:", err) + } + err = c.RemoveBucket(bucketName) + if err != nil { + logger().Fatal("Error:", err) + } +} + +// Tests FPutObject of a big file to trigger multipart +func testFPutObjectMultipart() { + logger().Info() + + // Seed random based on current time. + rand.Seed(time.Now().Unix()) + + // Instantiate new minio client object. + c, err := minio.New( + os.Getenv(serverEndpoint), + os.Getenv(accessKey), + os.Getenv(secretKey), + mustParseBool(os.Getenv(enableHTTPS)), + ) + if err != nil { + logger().Fatal("Error:", err) + } + + // Enable tracing, write to stderr. + // c.TraceOn(os.Stderr) + + // Set user agent. + c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + + // Generate a new random bucket name. + bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") + + // Make a new bucket. + err = c.MakeBucket(bucketName, "us-east-1") + if err != nil { + logger().Fatal("Error:", err, bucketName) + } + + // Upload 4 parts to utilize all 3 'workers' in multipart and still have a part to upload. + var fileName = getFilePath("datafile-65-MB") + if os.Getenv("MINT_DATA_DIR") == "" { + // Make a temp file with minPartSize bytes of data. + file, err := ioutil.TempFile(os.TempDir(), "FPutObjectTest") + if err != nil { + logger().Fatal("Error:", err) + } + + // Upload 4 parts to utilize all 3 'workers' in multipart and still have a part to upload. + var buffer = bytes.Repeat([]byte(string('a')), MinPartSize) + if _, err := file.Write(buffer); err != nil { + logger().Fatal("Error:", err) + } + // Close the file pro-actively for windows. + err = file.Close() + if err != nil { + logger().Fatal("Error:", err) + } + fileName = file.Name() + } + totalSize := MinPartSize * 1 + // Set base object name + objectName := bucketName + "FPutObject" + objectContentType := "testapplication/octet-stream" + + // Perform standard FPutObject with contentType provided (Expecting application/octet-stream) + n, err := c.FPutObject(bucketName, objectName+"-standard", fileName, objectContentType) + if err != nil { + logger().Fatal("Error:", err) + } + if n != int64(totalSize) { + logger().Fatalf("Error: number of bytes does not match, want %v, got %v\n", totalSize, n) + } + + r, err := c.GetObject(bucketName, objectName+"-standard") + if err != nil { + logger().Fatalf("Unexpected error: %v\n", err) + } + objInfo, err := r.Stat() + if err != nil { + logger().Fatalf("Unexpected error: %v\n", err) + } + if objInfo.Size != int64(totalSize) { + logger().Fatalf("Error: number of bytes does not match, want %v, got %v\n", totalSize, n) + } + if objInfo.ContentType != objectContentType { + logger().Fatalf("Error: Content types don't match, want %v, got %v\n", objectContentType, objInfo.ContentType) + } + + // Remove all objects and bucket and temp file + err = c.RemoveObject(bucketName, objectName+"-standard") + if err != nil { + logger().Fatal("Error: ", err) + } + + err = c.RemoveBucket(bucketName) + if err != nil { + logger().Fatal("Error:", err) + } +} + +// Tests FPutObject hidden contentType setting +func testFPutObject() { + logger().Info() + + // Seed random based on current time. + rand.Seed(time.Now().Unix()) + + // Instantiate new minio client object. + c, err := minio.New( + os.Getenv(serverEndpoint), + os.Getenv(accessKey), + os.Getenv(secretKey), + mustParseBool(os.Getenv(enableHTTPS)), + ) + if err != nil { + logger().Fatal("Error:", err) + } + + // Enable tracing, write to stderr. + // c.TraceOn(os.Stderr) + + // Set user agent. + c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + + // Generate a new random bucket name. + bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") + + // Make a new bucket. + err = c.MakeBucket(bucketName, "us-east-1") + if err != nil { + logger().Fatal("Error:", err, bucketName) + } + + // Upload 3 parts worth of data to use all 3 of multiparts 'workers' and have an extra part. + // Use different data in part for multipart tests to check parts are uploaded in correct order. + var fName = getFilePath("datafile-65-MB") + if os.Getenv("MINT_DATA_DIR") == "" { + // Make a temp file with minPartSize bytes of data. + file, err := ioutil.TempFile(os.TempDir(), "FPutObjectTest") + if err != nil { + logger().Fatal("Error:", err) + } + + // Upload 4 parts to utilize all 3 'workers' in multipart and still have a part to upload. + var buffer = bytes.Repeat([]byte(string('a')), MinPartSize) + if _, err = file.Write(buffer); err != nil { + logger().Fatal("Error:", err) + } + // Close the file pro-actively for windows. + err = file.Close() + if err != nil { + logger().Fatal("Error:", err) + } + fName = file.Name() + } + var totalSize = MinPartSize * 1 + + // Set base object name + objectName := bucketName + "FPutObject" + + // Perform standard FPutObject with contentType provided (Expecting application/octet-stream) + n, err := c.FPutObject(bucketName, objectName+"-standard", fName, "application/octet-stream") + if err != nil { + logger().Fatal("Error:", err) + } + if n != int64(totalSize) { + logger().Fatalf("Error: number of bytes does not match, want %v, got %v\n", totalSize, n) + } + + // Perform FPutObject with no contentType provided (Expecting application/octet-stream) + n, err = c.FPutObject(bucketName, objectName+"-Octet", fName, "") + if err != nil { + logger().Fatal("Error:", err) + } + if n != int64(totalSize) { + logger().Fatalf("Error: number of bytes does not match, want %v, got %v\n", totalSize, n) + } + srcFile, err := os.Open(fName) + if err != nil { + logger().Fatal("Error:", err) + } + defer srcFile.Close() + // Add extension to temp file name + tmpFile, err := os.Create(fName + ".gtar") + if err != nil { + logger().Fatal("Error:", err) + } + defer tmpFile.Close() + _, err = io.Copy(tmpFile, srcFile) + if err != nil { + logger().Fatal("Error:", err) + } + + // Perform FPutObject with no contentType provided (Expecting application/x-gtar) + n, err = c.FPutObject(bucketName, objectName+"-GTar", fName+".gtar", "") + if err != nil { + logger().Fatal("Error:", err) + } + if n != int64(totalSize) { + logger().Fatalf("Error: number of bytes does not match, want %v, got %v\n", totalSize, n) + } + + // Check headers + rStandard, err := c.StatObject(bucketName, objectName+"-standard") + if err != nil { + logger().Fatal("Error:", err, bucketName, objectName+"-standard") + } + if rStandard.ContentType != "application/octet-stream" { + logger().Fatalf("Error: Content-Type headers mismatched, want %v, got %v\n", + "application/octet-stream", rStandard.ContentType) + } + + rOctet, err := c.StatObject(bucketName, objectName+"-Octet") + if err != nil { + logger().Fatal("Error:", err, bucketName, objectName+"-Octet") + } + if rOctet.ContentType != "application/octet-stream" { + logger().Fatalf("Error: Content-Type headers mismatched, want %v, got %v\n", + "application/octet-stream", rStandard.ContentType) + } + + rGTar, err := c.StatObject(bucketName, objectName+"-GTar") + if err != nil { + logger().Fatal("Error:", err, bucketName, objectName+"-GTar") + } + if rGTar.ContentType != "application/x-gtar" { + logger().Fatalf("Error: Content-Type headers mismatched, want %v, got %v\n", + "application/x-gtar", rStandard.ContentType) + } + + // Remove all objects and bucket and temp file + err = c.RemoveObject(bucketName, objectName+"-standard") + if err != nil { + logger().Fatal("Error: ", err) + } + + err = c.RemoveObject(bucketName, objectName+"-Octet") + if err != nil { + logger().Fatal("Error: ", err) + } + + err = c.RemoveObject(bucketName, objectName+"-GTar") + if err != nil { + logger().Fatal("Error: ", err) + } + + err = c.RemoveBucket(bucketName) + if err != nil { + logger().Fatal("Error:", err) + } + + err = os.Remove(fName + ".gtar") + if err != nil { + logger().Fatal("Error:", err) + } + +} + +// Tests get object ReaderSeeker interface methods. +func testGetObjectReadSeekFunctional() { + logger().Info() + + // Seed random based on current time. + rand.Seed(time.Now().Unix()) + + // Instantiate new minio client object. + c, err := minio.New( + os.Getenv(serverEndpoint), + os.Getenv(accessKey), + os.Getenv(secretKey), + mustParseBool(os.Getenv(enableHTTPS)), + ) + if err != nil { + logger().Fatal("Error:", err) + } + + // Enable tracing, write to stderr. + // c.TraceOn(os.Stderr) + + // Set user agent. + c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + + // Generate a new random bucket name. + bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") + + // Make a new bucket. + err = c.MakeBucket(bucketName, "us-east-1") + if err != nil { + logger().Fatal("Error:", err, bucketName) + } + + // Generate data more than 32K + var buf = getDataBuffer("datafile-33-kB", rand.Intn(1<<20)+32*1024) + + bufSize := len(buf) + + // Save the data + objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") + n, err := c.PutObject(bucketName, objectName, bytes.NewReader(buf), "binary/octet-stream") + if err != nil { + logger().Fatal("Error:", err, bucketName, objectName) + } + + if n != int64(bufSize) { + logger().Fatalf("Error: number of bytes does not match, want %v, got %v\n", len(buf), n) + } + + defer func() { + err = c.RemoveObject(bucketName, objectName) + if err != nil { + logger().Fatal("Error: ", err) + } + err = c.RemoveBucket(bucketName) + if err != nil { + logger().Fatal("Error:", err) + } + }() + + // Read the data back + r, err := c.GetObject(bucketName, objectName) + if err != nil { + logger().Fatal("Error:", err, bucketName, objectName) + } + + st, err := r.Stat() + if err != nil { + logger().Fatal("Error:", err, bucketName, objectName) + } + if st.Size != int64(bufSize) { + logger().Fatalf("Error: number of bytes in stat does not match, want %v, got %v\n", + len(buf), st.Size) + } + + // This following function helps us to compare data from the reader after seek + // with the data from the original buffer + cmpData := func(r io.Reader, start, end int) { + if end-start == 0 { + return + } + buffer := bytes.NewBuffer([]byte{}) + if _, err := io.CopyN(buffer, r, int64(bufSize)); err != nil { + if err != io.EOF { + logger().Fatal("Error:", err) + } + } + if !bytes.Equal(buf[start:end], buffer.Bytes()) { + logger().Fatal("Error: Incorrect read bytes v/s original buffer.") + } + } + + // Generic seek error for errors other than io.EOF + seekErr := errors.New("seek error") + + testCases := []struct { + offset int64 + whence int + pos int64 + err error + shouldCmp bool + start int + end int + }{ + // Start from offset 0, fetch data and compare + {0, 0, 0, nil, true, 0, 0}, + // Start from offset 2048, fetch data and compare + {2048, 0, 2048, nil, true, 2048, bufSize}, + // Start from offset larger than possible + {int64(bufSize) + 1024, 0, 0, seekErr, false, 0, 0}, + // Move to offset 0 without comparing + {0, 0, 0, nil, false, 0, 0}, + // Move one step forward and compare + {1, 1, 1, nil, true, 1, bufSize}, + // Move larger than possible + {int64(bufSize), 1, 0, seekErr, false, 0, 0}, + // Provide negative offset with CUR_SEEK + {int64(-1), 1, 0, seekErr, false, 0, 0}, + // Test with whence SEEK_END and with positive offset + {1024, 2, int64(bufSize) - 1024, io.EOF, true, 0, 0}, + // Test with whence SEEK_END and with negative offset + {-1024, 2, int64(bufSize) - 1024, nil, true, bufSize - 1024, bufSize}, + // Test with whence SEEK_END and with large negative offset + {-int64(bufSize) * 2, 2, 0, seekErr, true, 0, 0}, + } + + for i, testCase := range testCases { + // Perform seek operation + n, err := r.Seek(testCase.offset, testCase.whence) + // We expect an error + if testCase.err == seekErr && err == nil { + logger().Fatalf("Test %d, unexpected err value: expected: %v, found: %v", i+1, testCase.err, err) + } + // We expect a specific error + if testCase.err != seekErr && testCase.err != err { + logger().Fatalf("Test %d, unexpected err value: expected: %v, found: %v", i+1, testCase.err, err) + } + // If we expect an error go to the next loop + if testCase.err != nil { + continue + } + // Check the returned seek pos + if n != testCase.pos { + logger().Fatalf("Test %d, error: number of bytes seeked does not match, want %v, got %v\n", i+1, + testCase.pos, n) + } + // Compare only if shouldCmp is activated + if testCase.shouldCmp { + cmpData(r, testCase.start, testCase.end) + } + } +} + +// Tests get object ReaderAt interface methods. +func testGetObjectReadAtFunctional() { + logger().Info() + + // Seed random based on current time. + rand.Seed(time.Now().Unix()) + + // Instantiate new minio client object. + c, err := minio.New( + os.Getenv(serverEndpoint), + os.Getenv(accessKey), + os.Getenv(secretKey), + mustParseBool(os.Getenv(enableHTTPS)), + ) + if err != nil { + logger().Fatal("Error:", err) + } + + // Enable tracing, write to stderr. + // c.TraceOn(os.Stderr) + + // Set user agent. + c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + + // Generate a new random bucket name. + bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") + + // Make a new bucket. + err = c.MakeBucket(bucketName, "us-east-1") + if err != nil { + logger().Fatal("Error:", err, bucketName) + } + + // Generate data more than 32K + var buf = getDataBuffer("datafile-33-kB", rand.Intn(1<<20)+32*1024) + + // Save the data + objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") + n, err := c.PutObject(bucketName, objectName, bytes.NewReader(buf), "binary/octet-stream") + if err != nil { + logger().Fatal("Error:", err, bucketName, objectName) + } + + if n != int64(len(buf)) { + logger().Fatalf("Error: number of bytes does not match, want %v, got %v\n", len(buf), n) + } + + // read the data back + r, err := c.GetObject(bucketName, objectName) + if err != nil { + logger().Fatal("Error:", err, bucketName, objectName) + } + offset := int64(2048) + + // read directly + buf1 := make([]byte, 512) + buf2 := make([]byte, 512) + buf3 := make([]byte, 512) + buf4 := make([]byte, 512) + + // Test readAt before stat is called. + m, err := r.ReadAt(buf1, offset) + if err != nil { + logger().Fatal("Error:", err, len(buf1), offset) + } + if m != len(buf1) { + logger().Fatalf("Error: ReadAt read shorter bytes before reaching EOF, want %v, got %v\n", m, len(buf1)) + } + if !bytes.Equal(buf1, buf[offset:offset+512]) { + logger().Fatal("Error: Incorrect read between two ReadAt from same offset.") + } + offset += 512 + + st, err := r.Stat() + if err != nil { + logger().Fatal("Error:", err, bucketName, objectName) + } + if st.Size != int64(len(buf)) { + logger().Fatalf("Error: number of bytes in stat does not match, want %v, got %v\n", + len(buf), st.Size) + } + + m, err = r.ReadAt(buf2, offset) + if err != nil { + logger().Fatal("Error:", err, st.Size, len(buf2), offset) + } + if m != len(buf2) { + logger().Fatalf("Error: ReadAt read shorter bytes before reaching EOF, want %v, got %v\n", m, len(buf2)) + } + if !bytes.Equal(buf2, buf[offset:offset+512]) { + logger().Fatal("Error: Incorrect read between two ReadAt from same offset.") + } + offset += 512 + m, err = r.ReadAt(buf3, offset) + if err != nil { + logger().Fatal("Error:", err, st.Size, len(buf3), offset) + } + if m != len(buf3) { + logger().Fatalf("Error: ReadAt read shorter bytes before reaching EOF, want %v, got %v\n", m, len(buf3)) + } + if !bytes.Equal(buf3, buf[offset:offset+512]) { + logger().Fatal("Error: Incorrect read between two ReadAt from same offset.") + } + offset += 512 + m, err = r.ReadAt(buf4, offset) + if err != nil { + logger().Fatal("Error:", err, st.Size, len(buf4), offset) + } + if m != len(buf4) { + logger().Fatalf("Error: ReadAt read shorter bytes before reaching EOF, want %v, got %v\n", m, len(buf4)) + } + if !bytes.Equal(buf4, buf[offset:offset+512]) { + logger().Fatal("Error: Incorrect read between two ReadAt from same offset.") + } + + buf5 := make([]byte, n) + // Read the whole object. + m, err = r.ReadAt(buf5, 0) + if err != nil { + if err != io.EOF { + logger().Fatal("Error:", err, len(buf5)) + } + } + if m != len(buf5) { + logger().Fatalf("Error: ReadAt read shorter bytes before reaching EOF, want %v, got %v\n", m, len(buf5)) + } + if !bytes.Equal(buf, buf5) { + logger().Fatal("Error: Incorrect data read in GetObject, than what was previously upoaded.") + } + + buf6 := make([]byte, n+1) + // Read the whole object and beyond. + _, err = r.ReadAt(buf6, 0) + if err != nil { + if err != io.EOF { + logger().Fatal("Error:", err, len(buf6)) + } + } + err = c.RemoveObject(bucketName, objectName) + if err != nil { + logger().Fatal("Error: ", err) + } + err = c.RemoveBucket(bucketName) + if err != nil { + logger().Fatal("Error:", err) + } +} + +// Test Presigned Post Policy +func testPresignedPostPolicy() { + logger().Info() + + // Seed random based on current time. + rand.Seed(time.Now().Unix()) + + // Instantiate new minio client object + c, err := minio.NewV4( + os.Getenv(serverEndpoint), + os.Getenv(accessKey), + os.Getenv(secretKey), + mustParseBool(os.Getenv(enableHTTPS)), + ) + if err != nil { + logger().Fatal("Error:", err) + } + + // Enable tracing, write to stderr. + // c.TraceOn(os.Stderr) + + // Set user agent. + c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + + // Generate a new random bucket name. + bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") + + // Make a new bucket in 'us-east-1' (source bucket). + err = c.MakeBucket(bucketName, "us-east-1") + if err != nil { + logger().Fatal("Error:", err, bucketName) + } + + // Generate data more than 32K + var buf = getDataBuffer("datafile-33-kB", rand.Intn(1<<20)+32*1024) + + // Save the data + objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") + n, err := c.PutObject(bucketName, objectName, bytes.NewReader(buf), "binary/octet-stream") + if err != nil { + logger().Fatal("Error:", err, bucketName, objectName) + } + + if n != int64(len(buf)) { + logger().Fatalf("Error: number of bytes does not match want %v, got %v", + len(buf), n) + } + + policy := minio.NewPostPolicy() + + if err := policy.SetBucket(""); err == nil { + logger().Fatalf("Error: %s", err) + } + if err := policy.SetKey(""); err == nil { + logger().Fatalf("Error: %s", err) + } + if err := policy.SetKeyStartsWith(""); err == nil { + logger().Fatalf("Error: %s", err) + } + if err := policy.SetExpires(time.Date(1, time.January, 1, 0, 0, 0, 0, time.UTC)); err == nil { + logger().Fatalf("Error: %s", err) + } + if err := policy.SetContentType(""); err == nil { + logger().Fatalf("Error: %s", err) + } + if err := policy.SetContentLengthRange(1024*1024, 1024); err == nil { + logger().Fatalf("Error: %s", err) + } + + policy.SetBucket(bucketName) + policy.SetKey(objectName) + policy.SetExpires(time.Now().UTC().AddDate(0, 0, 10)) // expires in 10 days + policy.SetContentType("image/png") + policy.SetContentLengthRange(1024, 1024*1024) + + _, _, err = c.PresignedPostPolicy(policy) + if err != nil { + logger().Fatal("Error:", err) + } + + policy = minio.NewPostPolicy() + + // Remove all objects and buckets + err = c.RemoveObject(bucketName, objectName) + if err != nil { + logger().Fatal("Error:", err) + } + + err = c.RemoveBucket(bucketName) + if err != nil { + logger().Fatal("Error:", err) + } +} + +// Tests copy object +func testCopyObject() { + logger().Info() + // Seed random based on current time. + rand.Seed(time.Now().Unix()) + + // Instantiate new minio client object + c, err := minio.NewV4( + os.Getenv(serverEndpoint), + os.Getenv(accessKey), + os.Getenv(secretKey), + mustParseBool(os.Getenv(enableHTTPS)), + ) + if err != nil { + logger().Fatal("Error:", err) + } + + // Enable tracing, write to stderr. + // c.TraceOn(os.Stderr) + + // Set user agent. + c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + + // Generate a new random bucket name. + bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") + + // Make a new bucket in 'us-east-1' (source bucket). + err = c.MakeBucket(bucketName, "us-east-1") + if err != nil { + logger().Fatal("Error:", err, bucketName) + } + + // Make a new bucket in 'us-east-1' (destination bucket). + err = c.MakeBucket(bucketName+"-copy", "us-east-1") + if err != nil { + logger().Fatal("Error:", err, bucketName+"-copy") + } + + // Generate data more than 32K + buf := bytes.Repeat([]byte("5"), rand.Intn(1<<20)+32*1024) + + // Save the data + objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") + n, err := c.PutObject(bucketName, objectName, bytes.NewReader(buf), "binary/octet-stream") + if err != nil { + logger().Fatal("Error:", err, bucketName, objectName) + } + + if n != int64(len(buf)) { + logger().Fatalf("Error: number of bytes does not match want %v, got %v", + len(buf), n) + } + + r, err := c.GetObject(bucketName, objectName) + if err != nil { + logger().Fatal("Error:", err) + } + // Check the various fields of source object against destination object. + objInfo, err := r.Stat() + if err != nil { + logger().Fatal("Error:", err) + } + + // Copy Source + src := minio.NewSourceInfo(bucketName, objectName, nil) + + // Set copy conditions. + + // All invalid conditions first. + err = src.SetModifiedSinceCond(time.Date(1, time.January, 1, 0, 0, 0, 0, time.UTC)) + if err == nil { + logger().Fatal("Error:", err) + } + err = src.SetUnmodifiedSinceCond(time.Date(1, time.January, 1, 0, 0, 0, 0, time.UTC)) + if err == nil { + logger().Fatal("Error:", err) + } + err = src.SetMatchETagCond("") + if err == nil { + logger().Fatal("Error:", err) + } + err = src.SetMatchETagExceptCond("") + if err == nil { + logger().Fatal("Error:", err) + } + + err = src.SetModifiedSinceCond(time.Date(2014, time.April, 0, 0, 0, 0, 0, time.UTC)) + if err != nil { + logger().Fatal("Error:", err) + } + err = src.SetMatchETagCond(objInfo.ETag) + if err != nil { + logger().Fatal("Error:", err) + } + + dst, err := minio.NewDestinationInfo(bucketName+"-copy", objectName+"-copy", nil, nil) + if err != nil { + logger().Fatal(err) + } + + // Perform the Copy + err = c.CopyObject(dst, src) + if err != nil { + logger().Fatal("Error:", err, bucketName+"-copy", objectName+"-copy") + } + + // Source object + reader, err := c.GetObject(bucketName, objectName) + if err != nil { + logger().Fatal("Error:", err) + } + // Destination object + readerCopy, err := c.GetObject(bucketName+"-copy", objectName+"-copy") + if err != nil { + logger().Fatal("Error:", err) + } + // Check the various fields of source object against destination object. + objInfo, err = reader.Stat() + if err != nil { + logger().Fatal("Error:", err) + } + objInfoCopy, err := readerCopy.Stat() + if err != nil { + logger().Fatal("Error:", err) + } + if objInfo.Size != objInfoCopy.Size { + logger().Fatalf("Error: number of bytes does not match, want %v, got %v\n", + objInfo.Size, objInfoCopy.Size) + } + + // CopyObject again but with wrong conditions + src = minio.NewSourceInfo(bucketName, objectName, nil) + err = src.SetUnmodifiedSinceCond(time.Date(2014, time.April, 0, 0, 0, 0, 0, time.UTC)) + if err != nil { + logger().Fatal("Error:", err) + } + err = src.SetMatchETagExceptCond(objInfo.ETag) + if err != nil { + logger().Fatal("Error:", err) + } + + // Perform the Copy which should fail + err = c.CopyObject(dst, src) + if err == nil { + logger().Fatal("Error:", err, bucketName+"-copy", objectName+"-copy should fail") + } + + // Remove all objects and buckets + err = c.RemoveObject(bucketName, objectName) + if err != nil { + logger().Fatal("Error:", err) + } + + err = c.RemoveObject(bucketName+"-copy", objectName+"-copy") + if err != nil { + logger().Fatal("Error:", err) + } + + err = c.RemoveBucket(bucketName) + if err != nil { + logger().Fatal("Error:", err) + } + + err = c.RemoveBucket(bucketName + "-copy") + if err != nil { + logger().Fatal("Error:", err) + } +} + +// TestEncryptionPutGet tests client side encryption +func testEncryptionPutGet() { + logger().Info() + + // Seed random based on current time. + rand.Seed(time.Now().Unix()) + + // Instantiate new minio client object. + c, err := minio.New( + os.Getenv(serverEndpoint), + os.Getenv(accessKey), + os.Getenv(secretKey), + mustParseBool(os.Getenv(enableHTTPS)), + ) + if err != nil { + logger().Fatal("Error:", err) + } + + // Enable tracing, write to stderr. + // c.TraceOn(os.Stderr) + + // Set user agent. + c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + + // Generate a new random bucket name. + bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") + + // Make a new bucket. + err = c.MakeBucket(bucketName, "us-east-1") + if err != nil { + logger().Fatal("Error:", err, bucketName) + } + + // Generate a symmetric key + symKey := encrypt.NewSymmetricKey([]byte("my-secret-key-00")) + + // Generate an assymmetric key from predefine public and private certificates + privateKey, err := hex.DecodeString( + "30820277020100300d06092a864886f70d0101010500048202613082025d" + + "0201000281810087b42ea73243a3576dc4c0b6fa245d339582dfdbddc20c" + + "bb8ab666385034d997210c54ba79275c51162a1221c3fb1a4c7c61131ca6" + + "5563b319d83474ef5e803fbfa7e52b889e1893b02586b724250de7ac6351" + + "cc0b7c638c980acec0a07020a78eed7eaa471eca4b92071394e061346c06" + + "15ccce2f465dee2080a89e43f29b5702030100010281801dd5770c3af8b3" + + "c85cd18cacad81a11bde1acfac3eac92b00866e142301fee565365aa9af4" + + "57baebf8bb7711054d071319a51dd6869aef3848ce477a0dc5f0dbc0c336" + + "5814b24c820491ae2bb3c707229a654427e03307fec683e6b27856688f08" + + "bdaa88054c5eeeb773793ff7543ee0fb0e2ad716856f2777f809ef7e6fa4" + + "41024100ca6b1edf89e8a8f93cce4b98c76c6990a09eb0d32ad9d3d04fbf" + + "0b026fa935c44f0a1c05dd96df192143b7bda8b110ec8ace28927181fd8c" + + "d2f17330b9b63535024100aba0260afb41489451baaeba423bee39bcbd1e" + + "f63dd44ee2d466d2453e683bf46d019a8baead3a2c7fca987988eb4d565e" + + "27d6be34605953f5034e4faeec9bdb0241009db2cb00b8be8c36710aff96" + + "6d77a6dec86419baca9d9e09a2b761ea69f7d82db2ae5b9aae4246599bb2" + + "d849684d5ab40e8802cfe4a2b358ad56f2b939561d2902404e0ead9ecafd" + + "bb33f22414fa13cbcc22a86bdf9c212ce1a01af894e3f76952f36d6c904c" + + "bd6a7e0de52550c9ddf31f1e8bfe5495f79e66a25fca5c20b3af5b870241" + + "0083456232aa58a8c45e5b110494599bda8dbe6a094683a0539ddd24e19d" + + "47684263bbe285ad953d725942d670b8f290d50c0bca3d1dc9688569f1d5" + + "9945cb5c7d") + + if err != nil { + logger().Fatal(err) + } + + publicKey, err := hex.DecodeString("30819f300d06092a864886f70d010101050003818d003081890281810087" + + "b42ea73243a3576dc4c0b6fa245d339582dfdbddc20cbb8ab666385034d9" + + "97210c54ba79275c51162a1221c3fb1a4c7c61131ca65563b319d83474ef" + + "5e803fbfa7e52b889e1893b02586b724250de7ac6351cc0b7c638c980ace" + + "c0a07020a78eed7eaa471eca4b92071394e061346c0615ccce2f465dee20" + + "80a89e43f29b570203010001") + if err != nil { + logger().Fatal(err) + } + + // Generate an asymmetric key + asymKey, err := encrypt.NewAsymmetricKey(privateKey, publicKey) + if err != nil { + logger().Fatal(err) + } + + // Object custom metadata + customContentType := "custom/contenttype" + + testCases := []struct { + buf []byte + encKey encrypt.Key + }{ + {encKey: symKey, buf: bytes.Repeat([]byte("F"), 0)}, + {encKey: symKey, buf: bytes.Repeat([]byte("F"), 1)}, + {encKey: symKey, buf: bytes.Repeat([]byte("F"), 15)}, + {encKey: symKey, buf: bytes.Repeat([]byte("F"), 16)}, + {encKey: symKey, buf: bytes.Repeat([]byte("F"), 17)}, + {encKey: symKey, buf: bytes.Repeat([]byte("F"), 31)}, + {encKey: symKey, buf: bytes.Repeat([]byte("F"), 32)}, + {encKey: symKey, buf: bytes.Repeat([]byte("F"), 33)}, + {encKey: symKey, buf: bytes.Repeat([]byte("F"), 1024)}, + {encKey: symKey, buf: bytes.Repeat([]byte("F"), 1024*2)}, + {encKey: symKey, buf: bytes.Repeat([]byte("F"), 1024*1024)}, + + {encKey: asymKey, buf: bytes.Repeat([]byte("F"), 0)}, + {encKey: asymKey, buf: bytes.Repeat([]byte("F"), 1)}, + {encKey: asymKey, buf: bytes.Repeat([]byte("F"), 16)}, + {encKey: asymKey, buf: bytes.Repeat([]byte("F"), 32)}, + {encKey: asymKey, buf: bytes.Repeat([]byte("F"), 1024)}, + {encKey: asymKey, buf: bytes.Repeat([]byte("F"), 1024*1024)}, + } + + for i, testCase := range testCases { + // Generate a random object name + objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") + + // Secured object + cbcMaterials, err := encrypt.NewCBCSecureMaterials(testCase.encKey) + if err != nil { + logger().Fatal(err) + } + + // Put encrypted data + _, err = c.PutEncryptedObject(bucketName, objectName, bytes.NewReader(testCase.buf), cbcMaterials, map[string][]string{"Content-Type": {customContentType}}, nil) + if err != nil { + logger().Fatalf("Test %d, error: %v %v %v", i+1, err, bucketName, objectName) + } + + // Read the data back + r, err := c.GetEncryptedObject(bucketName, objectName, cbcMaterials) + if err != nil { + logger().Fatalf("Test %d, error: %v %v %v", i+1, err, bucketName, objectName) + } + defer r.Close() + + // Compare the sent object with the received one + recvBuffer := bytes.NewBuffer([]byte{}) + if _, err = io.Copy(recvBuffer, r); err != nil { + logger().Fatalf("Test %d, error: %v", i+1, err) + } + if recvBuffer.Len() != len(testCase.buf) { + logger().Fatalf("Test %d, error: number of bytes of received object does not match, want %v, got %v\n", + i+1, len(testCase.buf), recvBuffer.Len()) + } + if !bytes.Equal(testCase.buf, recvBuffer.Bytes()) { + logger().Fatalf("Test %d, error: Encrypted sent is not equal to decrypted, want `%x`, go `%x`", i+1, testCase.buf, recvBuffer.Bytes()) + } + + // Remove test object + err = c.RemoveObject(bucketName, objectName) + if err != nil { + logger().Fatalf("Test %d, error: %v", i+1, err) + } + + } + + // Remove test bucket + err = c.RemoveBucket(bucketName) + if err != nil { + logger().Fatal("Error:", err) + } + +} + +func testBucketNotification() { + logger().Info() + + if os.Getenv("NOTIFY_BUCKET") == "" || + os.Getenv("NOTIFY_SERVICE") == "" || + os.Getenv("NOTIFY_REGION") == "" || + os.Getenv("NOTIFY_ACCOUNTID") == "" || + os.Getenv("NOTIFY_RESOURCE") == "" { + logger().Info("skipping notification test if not configured") + return + } + + // Seed random based on current time. + rand.Seed(time.Now().Unix()) + + c, err := minio.New( + os.Getenv(serverEndpoint), + os.Getenv(accessKey), + os.Getenv(secretKey), + mustParseBool(os.Getenv(enableHTTPS)), + ) + if err != nil { + logger().Fatal("Error:", err) + } + + // Enable to debug + // c.TraceOn(os.Stderr) + + // Set user agent. + c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + + bucketName := os.Getenv("NOTIFY_BUCKET") + + topicArn := minio.NewArn("aws", os.Getenv("NOTIFY_SERVICE"), os.Getenv("NOTIFY_REGION"), os.Getenv("NOTIFY_ACCOUNTID"), os.Getenv("NOTIFY_RESOURCE")) + queueArn := minio.NewArn("aws", "dummy-service", "dummy-region", "dummy-accountid", "dummy-resource") + + topicConfig := minio.NewNotificationConfig(topicArn) + topicConfig.AddEvents(minio.ObjectCreatedAll, minio.ObjectRemovedAll) + topicConfig.AddFilterSuffix("jpg") + + queueConfig := minio.NewNotificationConfig(queueArn) + queueConfig.AddEvents(minio.ObjectCreatedAll) + queueConfig.AddFilterPrefix("photos/") + + bNotification := minio.BucketNotification{} + bNotification.AddTopic(topicConfig) + + // Add the same topicConfig again, should have no effect + // because it is duplicated + bNotification.AddTopic(topicConfig) + if len(bNotification.TopicConfigs) != 1 { + logger().Fatal("Error: duplicated entry added") + } + + // Add and remove a queue config + bNotification.AddQueue(queueConfig) + bNotification.RemoveQueueByArn(queueArn) + + err = c.SetBucketNotification(bucketName, bNotification) + if err != nil { + logger().Fatal("Error: ", err) + } + + bNotification, err = c.GetBucketNotification(bucketName) + if err != nil { + logger().Fatal("Error: ", err) + } + + if len(bNotification.TopicConfigs) != 1 { + logger().Fatal("Error: Topic config is empty") + } + + if bNotification.TopicConfigs[0].Filter.S3Key.FilterRules[0].Value != "jpg" { + logger().Fatal("Error: cannot get the suffix") + } + + err = c.RemoveAllBucketNotification(bucketName) + if err != nil { + logger().Fatal("Error: cannot delete bucket notification") + } +} + +// Tests comprehensive list of all methods. +func testFunctional() { + logger().Info() + + // Seed random based on current time. + rand.Seed(time.Now().Unix()) + + c, err := minio.New( + os.Getenv(serverEndpoint), + os.Getenv(accessKey), + os.Getenv(secretKey), + mustParseBool(os.Getenv(enableHTTPS)), + ) + if err != nil { + logger().Fatal("Error:", err) + } + + // Enable to debug + // c.TraceOn(os.Stderr) + + // Set user agent. + c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + + // Generate a new random bucket name. + bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") + + // Make a new bucket. + err = c.MakeBucket(bucketName, "us-east-1") + if err != nil { + logger().Fatal("Error:", err, bucketName) + } + + // Generate a random file name. + fileName := randString(60, rand.NewSource(time.Now().UnixNano()), "") + file, err := os.Create(fileName) + if err != nil { + logger().Fatal("Error:", err) + } + for i := 0; i < 3; i++ { + buf := make([]byte, rand.Intn(1<<19)) + _, err = file.Write(buf) + if err != nil { + logger().Fatal("Error:", err) + } + } + file.Close() + + // Verify if bucket exits and you have access. + var exists bool + exists, err = c.BucketExists(bucketName) + if err != nil { + logger().Fatal("Error:", err, bucketName) + } + if !exists { + logger().Fatal("Error: could not find ", bucketName) + } + + // Asserting the default bucket policy. + policyAccess, err := c.GetBucketPolicy(bucketName, "") + if err != nil { + logger().Fatal("Error:", err) + } + if policyAccess != "none" { + logger().Fatalf("Default bucket policy incorrect") + } + // Set the bucket policy to 'public readonly'. + err = c.SetBucketPolicy(bucketName, "", policy.BucketPolicyReadOnly) + if err != nil { + logger().Fatal("Error:", err) + } + // should return policy `readonly`. + policyAccess, err = c.GetBucketPolicy(bucketName, "") + if err != nil { + logger().Fatal("Error:", err) + } + if policyAccess != "readonly" { + logger().Fatalf("Expected bucket policy to be readonly") + } + + // Make the bucket 'public writeonly'. + err = c.SetBucketPolicy(bucketName, "", policy.BucketPolicyWriteOnly) + if err != nil { + logger().Fatal("Error:", err) + } + // should return policy `writeonly`. + policyAccess, err = c.GetBucketPolicy(bucketName, "") + if err != nil { + logger().Fatal("Error:", err) + } + if policyAccess != "writeonly" { + logger().Fatalf("Expected bucket policy to be writeonly") + } + // Make the bucket 'public read/write'. + err = c.SetBucketPolicy(bucketName, "", policy.BucketPolicyReadWrite) + if err != nil { + logger().Fatal("Error:", err) + } + // should return policy `readwrite`. + policyAccess, err = c.GetBucketPolicy(bucketName, "") + if err != nil { + logger().Fatal("Error:", err) + } + if policyAccess != "readwrite" { + logger().Fatalf("Expected bucket policy to be readwrite") + } + // List all buckets. + buckets, err := c.ListBuckets() + if len(buckets) == 0 { + logger().Fatal("Error: list buckets cannot be empty", buckets) + } + if err != nil { + logger().Fatal("Error:", err) + } + + // Verify if previously created bucket is listed in list buckets. + bucketFound := false + for _, bucket := range buckets { + if bucket.Name == bucketName { + bucketFound = true + } + } + + // If bucket not found error out. + if !bucketFound { + logger().Fatal("Error: bucket ", bucketName, "not found") + } + + objectName := bucketName + "unique" + + // Generate data + buf := bytes.Repeat([]byte("f"), 1<<19) + + n, err := c.PutObject(bucketName, objectName, bytes.NewReader(buf), "") + if err != nil { + logger().Fatal("Error: ", err) + } + if n != int64(len(buf)) { + logger().Fatal("Error: bad length ", n, len(buf)) + } + + n, err = c.PutObject(bucketName, objectName+"-nolength", bytes.NewReader(buf), "binary/octet-stream") + if err != nil { + logger().Fatal("Error:", err, bucketName, objectName+"-nolength") + } + + if n != int64(len(buf)) { + logger().Fatalf("Error: number of bytes does not match, want %v, got %v\n", len(buf), n) + } + + // Instantiate a done channel to close all listing. + doneCh := make(chan struct{}) + defer close(doneCh) + + objFound := false + isRecursive := true // Recursive is true. + for obj := range c.ListObjects(bucketName, objectName, isRecursive, doneCh) { + if obj.Key == objectName { + objFound = true + break + } + } + if !objFound { + logger().Fatal("Error: object " + objectName + " not found.") + } + + objFound = false + isRecursive = true // Recursive is true. + for obj := range c.ListObjectsV2(bucketName, objectName, isRecursive, doneCh) { + if obj.Key == objectName { + objFound = true + break + } + } + if !objFound { + logger().Fatal("Error: object " + objectName + " not found.") + } + + incompObjNotFound := true + for objIncompl := range c.ListIncompleteUploads(bucketName, objectName, isRecursive, doneCh) { + if objIncompl.Key != "" { + incompObjNotFound = false + break + } + } + if !incompObjNotFound { + logger().Fatal("Error: unexpected dangling incomplete upload found.") + } + + newReader, err := c.GetObject(bucketName, objectName) + if err != nil { + logger().Fatal("Error: ", err) + } + + newReadBytes, err := ioutil.ReadAll(newReader) + if err != nil { + logger().Fatal("Error: ", err) + } + + if !bytes.Equal(newReadBytes, buf) { + logger().Fatal("Error: bytes mismatch.") + } + + err = c.FGetObject(bucketName, objectName, fileName+"-f") + if err != nil { + logger().Fatal("Error: ", err) + } + + // Generate presigned GET object url. + presignedGetURL, err := c.PresignedGetObject(bucketName, objectName, 3600*time.Second, nil) + if err != nil { + logger().Fatal("Error: ", err) + } + + // Verify if presigned url works. + resp, err := http.Get(presignedGetURL.String()) + if err != nil { + logger().Fatal("Error: ", err) + } + if resp.StatusCode != http.StatusOK { + logger().Fatal("Error: ", resp.Status) + } + newPresignedBytes, err := ioutil.ReadAll(resp.Body) + if err != nil { + logger().Fatal("Error: ", err) + } + if !bytes.Equal(newPresignedBytes, buf) { + logger().Fatal("Error: bytes mismatch.") + } + + // Set request parameters. + reqParams := make(url.Values) + reqParams.Set("response-content-disposition", "attachment; filename=\"test.txt\"") + presignedGetURL, err = c.PresignedGetObject(bucketName, objectName, 3600*time.Second, reqParams) + if err != nil { + logger().Fatal("Error: ", err) + } + // Verify if presigned url works. + resp, err = http.Get(presignedGetURL.String()) + if err != nil { + logger().Fatal("Error: ", err) + } + if resp.StatusCode != http.StatusOK { + logger().Fatal("Error: ", resp.Status) + } + newPresignedBytes, err = ioutil.ReadAll(resp.Body) + if err != nil { + logger().Fatal("Error: ", err) + } + if !bytes.Equal(newPresignedBytes, buf) { + logger().Fatal("Error: bytes mismatch for presigned GET URL.") + } + if resp.Header.Get("Content-Disposition") != "attachment; filename=\"test.txt\"" { + logger().Fatalf("Error: wrong Content-Disposition received %s", resp.Header.Get("Content-Disposition")) + } + + presignedPutURL, err := c.PresignedPutObject(bucketName, objectName+"-presigned", 3600*time.Second) + if err != nil { + logger().Fatal("Error: ", err) + } + + buf = bytes.Repeat([]byte("g"), 1<<19) + + req, err := http.NewRequest("PUT", presignedPutURL.String(), bytes.NewReader(buf)) + if err != nil { + logger().Fatal("Error: ", err) + } + httpClient := &http.Client{ + // Setting a sensible time out of 30secs to wait for response + // headers. Request is pro-actively cancelled after 30secs + // with no response. + Timeout: 30 * time.Second, + Transport: http.DefaultTransport, + } + resp, err = httpClient.Do(req) + if err != nil { + logger().Fatal("Error: ", err) + } + + newReader, err = c.GetObject(bucketName, objectName+"-presigned") + if err != nil { + logger().Fatal("Error: ", err) + } + + newReadBytes, err = ioutil.ReadAll(newReader) + if err != nil { + logger().Fatal("Error: ", err) + } + + if !bytes.Equal(newReadBytes, buf) { + logger().Fatal("Error: bytes mismatch.") + } + + err = c.RemoveObject(bucketName, objectName) + if err != nil { + logger().Fatal("Error: ", err) + } + err = c.RemoveObject(bucketName, objectName+"-f") + if err != nil { + logger().Fatal("Error: ", err) + } + err = c.RemoveObject(bucketName, objectName+"-nolength") + if err != nil { + logger().Fatal("Error: ", err) + } + err = c.RemoveObject(bucketName, objectName+"-presigned") + if err != nil { + logger().Fatal("Error: ", err) + } + err = c.RemoveBucket(bucketName) + if err != nil { + logger().Fatal("Error:", err) + } + err = c.RemoveBucket(bucketName) + if err == nil { + logger().Fatal("Error:") + } + if err.Error() != "The specified bucket does not exist" { + logger().Fatal("Error: ", err) + } + if err = os.Remove(fileName); err != nil { + logger().Fatal("Error: ", err) + } + if err = os.Remove(fileName + "-f"); err != nil { + logger().Fatal("Error: ", err) + } +} + +// Test for validating GetObject Reader* methods functioning when the +// object is modified in the object store. +func testGetObjectObjectModified() { + logger().Info() + + // Instantiate new minio client object. + c, err := minio.NewV4( + os.Getenv(serverEndpoint), + os.Getenv(accessKey), + os.Getenv(secretKey), + mustParseBool(os.Getenv(enableHTTPS)), + ) + if err != nil { + logger().Fatal("Error:", err) + } + + // Enable tracing, write to stderr. + // c.TraceOn(os.Stderr) + + // Set user agent. + c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + + // Make a new bucket. + bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") + err = c.MakeBucket(bucketName, "us-east-1") + if err != nil { + logger().Fatal("Error:", err, bucketName) + } + defer c.RemoveBucket(bucketName) + + // Upload an object. + objectName := "myobject" + content := "helloworld" + _, err = c.PutObject(bucketName, objectName, strings.NewReader(content), "application/text") + if err != nil { + logger().Fatalf("Failed to upload %s/%s: %v", bucketName, objectName, err) + } + + defer c.RemoveObject(bucketName, objectName) + + reader, err := c.GetObject(bucketName, objectName) + if err != nil { + logger().Fatalf("Failed to get object %s/%s: %v", bucketName, objectName, err) + } + defer reader.Close() + + // Read a few bytes of the object. + b := make([]byte, 5) + n, err := reader.ReadAt(b, 0) + if err != nil { + logger().Fatalf("Failed to read object %s/%s at an offset: %v", bucketName, objectName, err) + } + + // Upload different contents to the same object while object is being read. + newContent := "goodbyeworld" + _, err = c.PutObject(bucketName, objectName, strings.NewReader(newContent), "application/text") + if err != nil { + logger().Fatalf("Failed to upload %s/%s: %v", bucketName, objectName, err) + } + + // Confirm that a Stat() call in between doesn't change the Object's cached etag. + _, err = reader.Stat() + if err.Error() != "At least one of the pre-conditions you specified did not hold" { + log.Error(fmt.Errorf("Expected Stat to fail with error %s but received %s", "At least one of the pre-conditions you specified did not hold", err.Error())) + } + + // Read again only to find object contents have been modified since last read. + _, err = reader.ReadAt(b, int64(n)) + if err.Error() != "At least one of the pre-conditions you specified did not hold" { + log.Error(fmt.Errorf("Expected ReadAt to fail with error %s but received %s", "At least one of the pre-conditions you specified did not hold", err.Error())) + } +} + +// Test validates putObject to upload a file seeked at a given offset. +func testPutObjectUploadSeekedObject() { + logger().Info() + + // Instantiate new minio client object. + c, err := minio.NewV4( + os.Getenv(serverEndpoint), + os.Getenv(accessKey), + os.Getenv(secretKey), + mustParseBool(os.Getenv(enableHTTPS)), + ) + if err != nil { + logger().Fatal("Error:", err) + } + + // Enable tracing, write to stderr. + // c.TraceOn(os.Stderr) + + // Set user agent. + c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + + // Make a new bucket. + bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") + err = c.MakeBucket(bucketName, "us-east-1") + if err != nil { + logger().Fatal("Error:", err, bucketName) + } + defer c.RemoveBucket(bucketName) + + tempfile, err := ioutil.TempFile("", "minio-go-upload-test-") + if err != nil { + logger().Fatal("Error:", err) + } + + var data []byte + if fileName := getFilePath("datafile-100-kB"); fileName != "" { + data, _ = ioutil.ReadFile(fileName) + } else { + // Generate data more than 32K + data = bytes.Repeat([]byte("1"), 120000) + } + var length = len(data) + if _, err = tempfile.Write(data); err != nil { + logger().Fatal("Error:", err) + } + + objectName := fmt.Sprintf("test-file-%v", rand.Uint32()) + + offset := length / 2 + if _, err := tempfile.Seek(int64(offset), 0); err != nil { + logger().Fatal("Error:", err) + } + + n, err := c.PutObject(bucketName, objectName, tempfile, "binary/octet-stream") + if err != nil { + logger().Fatal("Error:", err) + } + if n != int64(length-offset) { + logger().Fatalf("Invalid length returned, want %v, got %v", int64(length-offset), n) + } + tempfile.Close() + if err = os.Remove(tempfile.Name()); err != nil { + logger().Fatal("Error:", err) + } + + length = int(n) + + obj, err := c.GetObject(bucketName, objectName) + if err != nil { + logger().Fatal("Error:", err) + } + + n, err = obj.Seek(int64(offset), 0) + if err != nil { + logger().Fatal("Error:", err) + } + if n != int64(offset) { + logger().Fatalf("Invalid offset returned, want %v, got %v", int64(offset), n) + } + + n, err = c.PutObject(bucketName, objectName+"getobject", obj, "binary/octet-stream") + if err != nil { + logger().Fatal("Error:", err) + } + if n != int64(length-offset) { + logger().Fatalf("Invalid length returned, want %v, got %v", int64(length-offset), n) + } + + if err = c.RemoveObject(bucketName, objectName); err != nil { + logger().Fatal("Error:", err) + } + + if err = c.RemoveObject(bucketName, objectName+"getobject"); err != nil { + logger().Fatal("Error:", err) + } +} + +// Tests bucket re-create errors. +func testMakeBucketErrorV2() { + logger().Info() + if os.Getenv(serverEndpoint) != "s3.amazonaws.com" { + logger().Info("skipping region functional tests for non s3 runs") + return + } + + // Seed random based on current time. + rand.Seed(time.Now().Unix()) + + // Instantiate new minio client object. + c, err := minio.NewV2( + os.Getenv(serverEndpoint), + os.Getenv(accessKey), + os.Getenv(secretKey), + mustParseBool(os.Getenv(enableHTTPS)), + ) + if err != nil { + logger().Fatal("Error:", err) + } + + // Enable tracing, write to stderr. + // c.TraceOn(os.Stderr) + + // Set user agent. + c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + + // Generate a new random bucket name. + bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") + + // Make a new bucket in 'eu-west-1'. + if err = c.MakeBucket(bucketName, "eu-west-1"); err != nil { + logger().Fatal("Error:", err, bucketName) + } + if err = c.MakeBucket(bucketName, "eu-west-1"); err == nil { + logger().Fatal("Error: make bucket should should fail for", bucketName) + } + // Verify valid error response from server. + if minio.ToErrorResponse(err).Code != "BucketAlreadyExists" && + minio.ToErrorResponse(err).Code != "BucketAlreadyOwnedByYou" { + logger().Fatal("Error: Invalid error returned by server", err) + } + if err = c.RemoveBucket(bucketName); err != nil { + logger().Fatal("Error:", err, bucketName) + } +} + +// Test get object reader to not throw error on being closed twice. +func testGetObjectClosedTwiceV2() { + logger().Info() + + // Seed random based on current time. + rand.Seed(time.Now().Unix()) + + // Instantiate new minio client object. + c, err := minio.NewV2( + os.Getenv(serverEndpoint), + os.Getenv(accessKey), + os.Getenv(secretKey), + mustParseBool(os.Getenv(enableHTTPS)), + ) + if err != nil { + logger().Fatal("Error:", err) + } + + // Enable tracing, write to stderr. + // c.TraceOn(os.Stderr) + + // Set user agent. + c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + + // Generate a new random bucket name. + bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") + + // Make a new bucket. + err = c.MakeBucket(bucketName, "us-east-1") + if err != nil { + logger().Fatal("Error:", err, bucketName) + } + + // Generate data more than 32K. + var buf = getDataBuffer("datafile-33-kB", rand.Intn(1<<20)+32*1024) + + // Save the data + objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") + n, err := c.PutObject(bucketName, objectName, bytes.NewReader(buf), "binary/octet-stream") + if err != nil { + logger().Fatal("Error:", err, bucketName, objectName) + } + + if n != int64(len(buf)) { + logger().Fatalf("Error: number of bytes does not match, want %v, got %v\n", len(buf), n) + } + + // Read the data back + r, err := c.GetObject(bucketName, objectName) + if err != nil { + logger().Fatal("Error:", err, bucketName, objectName) + } + + st, err := r.Stat() + if err != nil { + logger().Fatal("Error:", err, bucketName, objectName) + } + if st.Size != int64(len(buf)) { + logger().Fatalf("Error: number of bytes in stat does not match, want %v, got %v\n", + len(buf), st.Size) + } + if err := r.Close(); err != nil { + logger().Fatal("Error:", err) + } + if err := r.Close(); err == nil { + logger().Fatal("Error: object is already closed, should return error") + } + + err = c.RemoveObject(bucketName, objectName) + if err != nil { + logger().Fatal("Error: ", err) + } + err = c.RemoveBucket(bucketName) + if err != nil { + logger().Fatal("Error:", err) + } +} + +// Tests removing partially uploaded objects. +func testRemovePartiallyUploadedV2() { + logger().Info() + + // Seed random based on current time. + rand.Seed(time.Now().Unix()) + + // Instantiate new minio client object. + c, err := minio.NewV2( + os.Getenv(serverEndpoint), + os.Getenv(accessKey), + os.Getenv(secretKey), + mustParseBool(os.Getenv(enableHTTPS)), + ) + if err != nil { + logger().Fatal("Error:", err) + } + + // Set user agent. + c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + + // Enable tracing, write to stdout. + // c.TraceOn(os.Stderr) + + // Generate a new random bucket name. + bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") + + // make a new bucket. + err = c.MakeBucket(bucketName, "us-east-1") + if err != nil { + logger().Fatal("Error:", err, bucketName) + } + + r := bytes.NewReader(bytes.Repeat([]byte("a"), 128*1024)) + + reader, writer := io.Pipe() + go func() { + i := 0 + for i < 25 { + _, cerr := io.CopyN(writer, r, 128*1024) + if cerr != nil { + logger().Fatal("Error:", cerr, bucketName) + } + i++ + r.Seek(0, 0) + } + writer.CloseWithError(errors.New("proactively closed to be verified later")) + }() + + objectName := bucketName + "-resumable" + _, err = c.PutObject(bucketName, objectName, reader, "application/octet-stream") + if err == nil { + logger().Fatal("Error: PutObject should fail.") + } + if err.Error() != "proactively closed to be verified later" { + logger().Fatal("Error:", err) + } + err = c.RemoveIncompleteUpload(bucketName, objectName) + if err != nil { + logger().Fatal("Error:", err) + } + err = c.RemoveBucket(bucketName) + if err != nil { + logger().Fatal("Error:", err) + } +} + +// Tests FPutObject hidden contentType setting +func testFPutObjectV2() { + logger().Info() + + // Seed random based on current time. + rand.Seed(time.Now().Unix()) + + // Instantiate new minio client object. + c, err := minio.NewV2( + os.Getenv(serverEndpoint), + os.Getenv(accessKey), + os.Getenv(secretKey), + mustParseBool(os.Getenv(enableHTTPS)), + ) + if err != nil { + logger().Fatal("Error:", err) + } + + // Enable tracing, write to stderr. + // c.TraceOn(os.Stderr) + + // Set user agent. + c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + + // Generate a new random bucket name. + bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") + + // Make a new bucket. + err = c.MakeBucket(bucketName, "us-east-1") + if err != nil { + logger().Fatal("Error:", err, bucketName) + } + + // Make a temp file with 11*1024*1024 bytes of data. + file, err := ioutil.TempFile(os.TempDir(), "FPutObjectTest") + if err != nil { + logger().Fatal("Error:", err) + } + + r := bytes.NewReader(bytes.Repeat([]byte("b"), 11*1024*1024)) + n, err := io.CopyN(file, r, 11*1024*1024) + if err != nil { + logger().Fatal("Error:", err) + } + if n != int64(11*1024*1024) { + logger().Fatalf("Error: number of bytes does not match, want %v, got %v\n", 11*1024*1024, n) + } + + // Close the file pro-actively for windows. + err = file.Close() + if err != nil { + logger().Fatal("Error:", err) + } + + // Set base object name + objectName := bucketName + "FPutObject" + + // Perform standard FPutObject with contentType provided (Expecting application/octet-stream) + n, err = c.FPutObject(bucketName, objectName+"-standard", file.Name(), "application/octet-stream") + if err != nil { + logger().Fatal("Error:", err) + } + if n != int64(11*1024*1024) { + logger().Fatalf("Error: number of bytes does not match, want %v, got %v\n", 11*1024*1024, n) + } + + // Perform FPutObject with no contentType provided (Expecting application/octet-stream) + n, err = c.FPutObject(bucketName, objectName+"-Octet", file.Name(), "") + if err != nil { + logger().Fatal("Error:", err) + } + if n != int64(11*1024*1024) { + logger().Fatalf("Error: number of bytes does not match, want %v, got %v\n", 11*1024*1024, n) + } + + // Add extension to temp file name + fileName := file.Name() + err = os.Rename(file.Name(), fileName+".gtar") + if err != nil { + logger().Fatal("Error:", err) + } + + // Perform FPutObject with no contentType provided (Expecting application/x-gtar) + n, err = c.FPutObject(bucketName, objectName+"-GTar", fileName+".gtar", "") + if err != nil { + logger().Fatal("Error:", err) + } + if n != int64(11*1024*1024) { + logger().Fatalf("Error: number of bytes does not match, want %v, got %v\n", 11*1024*1024, n) + } + + // Check headers + rStandard, err := c.StatObject(bucketName, objectName+"-standard") + if err != nil { + logger().Fatal("Error:", err, bucketName, objectName+"-standard") + } + if rStandard.ContentType != "application/octet-stream" { + logger().Fatalf("Error: Content-Type headers mismatched, want %v, got %v\n", + "application/octet-stream", rStandard.ContentType) + } + + rOctet, err := c.StatObject(bucketName, objectName+"-Octet") + if err != nil { + logger().Fatal("Error:", err, bucketName, objectName+"-Octet") + } + if rOctet.ContentType != "application/octet-stream" { + logger().Fatalf("Error: Content-Type headers mismatched, want %v, got %v\n", + "application/octet-stream", rStandard.ContentType) + } + + rGTar, err := c.StatObject(bucketName, objectName+"-GTar") + if err != nil { + logger().Fatal("Error:", err, bucketName, objectName+"-GTar") + } + if rGTar.ContentType != "application/x-gtar" { + logger().Fatalf("Error: Content-Type headers mismatched, want %v, got %v\n", + "application/x-gtar", rStandard.ContentType) + } + + // Remove all objects and bucket and temp file + err = c.RemoveObject(bucketName, objectName+"-standard") + if err != nil { + logger().Fatal("Error: ", err) + } + + err = c.RemoveObject(bucketName, objectName+"-Octet") + if err != nil { + logger().Fatal("Error: ", err) + } + + err = c.RemoveObject(bucketName, objectName+"-GTar") + if err != nil { + logger().Fatal("Error: ", err) + } + + err = c.RemoveBucket(bucketName) + if err != nil { + logger().Fatal("Error:", err) + } + + err = os.Remove(fileName + ".gtar") + if err != nil { + logger().Fatal("Error:", err) + } + +} + +// Tests various bucket supported formats. +func testMakeBucketRegionsV2() { + logger().Info() + if os.Getenv(serverEndpoint) != "s3.amazonaws.com" { + logger().Info("skipping region functional tests for non s3 runs") + return + } + + // Seed random based on current time. + rand.Seed(time.Now().Unix()) + + // Instantiate new minio client object. + c, err := minio.NewV2( + os.Getenv(serverEndpoint), + os.Getenv(accessKey), + os.Getenv(secretKey), + mustParseBool(os.Getenv(enableHTTPS)), + ) + if err != nil { + logger().Fatal("Error:", err) + } + + // Enable tracing, write to stderr. + // c.TraceOn(os.Stderr) + + // Set user agent. + c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + + // Generate a new random bucket name. + bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") + + // Make a new bucket in 'eu-central-1'. + if err = c.MakeBucket(bucketName, "eu-west-1"); err != nil { + logger().Fatal("Error:", err, bucketName) + } + + if err = c.RemoveBucket(bucketName); err != nil { + logger().Fatal("Error:", err, bucketName) + } + + // Make a new bucket with '.' in its name, in 'us-west-2'. This + // request is internally staged into a path style instead of + // virtual host style. + if err = c.MakeBucket(bucketName+".withperiod", "us-west-2"); err != nil { + logger().Fatal("Error:", err, bucketName+".withperiod") + } + + // Remove the newly created bucket. + if err = c.RemoveBucket(bucketName + ".withperiod"); err != nil { + logger().Fatal("Error:", err, bucketName+".withperiod") + } +} + +// Tests get object ReaderSeeker interface methods. +func testGetObjectReadSeekFunctionalV2() { + logger().Info() + // Seed random based on current time. + rand.Seed(time.Now().Unix()) + + // Instantiate new minio client object. + c, err := minio.NewV2( + os.Getenv(serverEndpoint), + os.Getenv(accessKey), + os.Getenv(secretKey), + mustParseBool(os.Getenv(enableHTTPS)), + ) + if err != nil { + logger().Fatal("Error:", err) + } + + // Enable tracing, write to stderr. + // c.TraceOn(os.Stderr) + + // Set user agent. + c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + + // Generate a new random bucket name. + bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") + + // Make a new bucket. + err = c.MakeBucket(bucketName, "us-east-1") + if err != nil { + logger().Fatal("Error:", err, bucketName) + } + + // Generate data more than 32K. + var buf = getDataBuffer("datafile-33-kB", rand.Intn(1<<20)+32*1024) + + // Save the data. + objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") + n, err := c.PutObject(bucketName, objectName, bytes.NewReader(buf), "binary/octet-stream") + if err != nil { + logger().Fatal("Error:", err, bucketName, objectName) + } + + if n != int64(len(buf)) { + logger().Fatalf("Error: number of bytes does not match, want %v, got %v\n", len(buf), n) + } + + // Read the data back + r, err := c.GetObject(bucketName, objectName) + if err != nil { + logger().Fatal("Error:", err, bucketName, objectName) + } + + st, err := r.Stat() + if err != nil { + logger().Fatal("Error:", err, bucketName, objectName) + } + if st.Size != int64(len(buf)) { + logger().Fatalf("Error: number of bytes in stat does not match, want %v, got %v\n", + len(buf), st.Size) + } + + offset := int64(2048) + n, err = r.Seek(offset, 0) + if err != nil { + logger().Fatal("Error:", err, offset) + } + if n != offset { + logger().Fatalf("Error: number of bytes seeked does not match, want %v, got %v\n", + offset, n) + } + n, err = r.Seek(0, 1) + if err != nil { + logger().Fatal("Error:", err) + } + if n != offset { + logger().Fatalf("Error: number of current seek does not match, want %v, got %v\n", + offset, n) + } + _, err = r.Seek(offset, 2) + if err == nil { + logger().Fatal("Error: seek on positive offset for whence '2' should error out") + } + n, err = r.Seek(-offset, 2) + if err != nil { + logger().Fatal("Error:", err) + } + if n != st.Size-offset { + logger().Fatalf("Error: number of bytes seeked back does not match, want %d, got %v\n", st.Size-offset, n) + } + + var buffer1 bytes.Buffer + if _, err = io.CopyN(&buffer1, r, st.Size); err != nil { + if err != io.EOF { + logger().Fatal("Error:", err) + } + } + if !bytes.Equal(buf[len(buf)-int(offset):], buffer1.Bytes()) { + logger().Fatal("Error: Incorrect read bytes v/s original buffer.") + } + + // Seek again and read again. + n, err = r.Seek(offset-1, 0) + if err != nil { + logger().Fatal("Error:", err) + } + if n != (offset - 1) { + logger().Fatalf("Error: number of bytes seeked back does not match, want %v, got %v\n", offset-1, n) + } + + var buffer2 bytes.Buffer + if _, err = io.CopyN(&buffer2, r, st.Size); err != nil { + if err != io.EOF { + logger().Fatal("Error:", err) + } + } + // Verify now lesser bytes. + if !bytes.Equal(buf[2047:], buffer2.Bytes()) { + logger().Fatal("Error: Incorrect read bytes v/s original buffer.") + } + + err = c.RemoveObject(bucketName, objectName) + if err != nil { + logger().Fatal("Error: ", err) + } + err = c.RemoveBucket(bucketName) + if err != nil { + logger().Fatal("Error:", err) + } +} + +// Tests get object ReaderAt interface methods. +func testGetObjectReadAtFunctionalV2() { + logger().Info() + // Seed random based on current time. + rand.Seed(time.Now().Unix()) + + // Instantiate new minio client object. + c, err := minio.NewV2( + os.Getenv(serverEndpoint), + os.Getenv(accessKey), + os.Getenv(secretKey), + mustParseBool(os.Getenv(enableHTTPS)), + ) + if err != nil { + logger().Fatal("Error:", err) + } + + // Enable tracing, write to stderr. + // c.TraceOn(os.Stderr) + + // Set user agent. + c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + + // Generate a new random bucket name. + bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") + + // Make a new bucket. + err = c.MakeBucket(bucketName, "us-east-1") + if err != nil { + logger().Fatal("Error:", err, bucketName) + } + + // Generate data more than 32K + var buf = getDataBuffer("datafile-33-kB", rand.Intn(1<<20)+32*1024) + + // Save the data + objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") + n, err := c.PutObject(bucketName, objectName, bytes.NewReader(buf), "binary/octet-stream") + if err != nil { + logger().Fatal("Error:", err, bucketName, objectName) + } + + if n != int64(len(buf)) { + logger().Fatalf("Error: number of bytes does not match, want %v, got %v\n", len(buf), n) + } + + // Read the data back + r, err := c.GetObject(bucketName, objectName) + if err != nil { + logger().Fatal("Error:", err, bucketName, objectName) + } + + st, err := r.Stat() + if err != nil { + logger().Fatal("Error:", err, bucketName, objectName) + } + if st.Size != int64(len(buf)) { + logger().Fatalf("Error: number of bytes in stat does not match, want %v, got %v\n", + len(buf), st.Size) + } + + offset := int64(2048) + + // Read directly + buf2 := make([]byte, 512) + buf3 := make([]byte, 512) + buf4 := make([]byte, 512) + + m, err := r.ReadAt(buf2, offset) + if err != nil { + logger().Fatal("Error:", err, st.Size, len(buf2), offset) + } + if m != len(buf2) { + logger().Fatalf("Error: ReadAt read shorter bytes before reaching EOF, want %v, got %v\n", m, len(buf2)) + } + if !bytes.Equal(buf2, buf[offset:offset+512]) { + logger().Fatal("Error: Incorrect read between two ReadAt from same offset.") + } + offset += 512 + m, err = r.ReadAt(buf3, offset) + if err != nil { + logger().Fatal("Error:", err, st.Size, len(buf3), offset) + } + if m != len(buf3) { + logger().Fatalf("Error: ReadAt read shorter bytes before reaching EOF, want %v, got %v\n", m, len(buf3)) + } + if !bytes.Equal(buf3, buf[offset:offset+512]) { + logger().Fatal("Error: Incorrect read between two ReadAt from same offset.") + } + offset += 512 + m, err = r.ReadAt(buf4, offset) + if err != nil { + logger().Fatal("Error:", err, st.Size, len(buf4), offset) + } + if m != len(buf4) { + logger().Fatalf("Error: ReadAt read shorter bytes before reaching EOF, want %v, got %v\n", m, len(buf4)) + } + if !bytes.Equal(buf4, buf[offset:offset+512]) { + logger().Fatal("Error: Incorrect read between two ReadAt from same offset.") + } + + buf5 := make([]byte, n) + // Read the whole object. + m, err = r.ReadAt(buf5, 0) + if err != nil { + if err != io.EOF { + logger().Fatal("Error:", err, len(buf5)) + } + } + if m != len(buf5) { + logger().Fatalf("Error: ReadAt read shorter bytes before reaching EOF, want %v, got %v\n", m, len(buf5)) + } + if !bytes.Equal(buf, buf5) { + logger().Fatal("Error: Incorrect data read in GetObject, than what was previously upoaded.") + } + + buf6 := make([]byte, n+1) + // Read the whole object and beyond. + _, err = r.ReadAt(buf6, 0) + if err != nil { + if err != io.EOF { + logger().Fatal("Error:", err, len(buf6)) + } + } + err = c.RemoveObject(bucketName, objectName) + if err != nil { + logger().Fatal("Error: ", err) + } + err = c.RemoveBucket(bucketName) + if err != nil { + logger().Fatal("Error:", err) + } +} + +// Tests copy object +func testCopyObjectV2() { + logger().Info() + + // Seed random based on current time. + rand.Seed(time.Now().Unix()) + + // Instantiate new minio client object + c, err := minio.NewV2( + os.Getenv(serverEndpoint), + os.Getenv(accessKey), + os.Getenv(secretKey), + mustParseBool(os.Getenv(enableHTTPS)), + ) + if err != nil { + logger().Fatal("Error:", err) + } + + // Enable tracing, write to stderr. + // c.TraceOn(os.Stderr) + + // Set user agent. + c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + + // Generate a new random bucket name. + bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") + + // Make a new bucket in 'us-east-1' (source bucket). + err = c.MakeBucket(bucketName, "us-east-1") + if err != nil { + logger().Fatal("Error:", err, bucketName) + } + + // Make a new bucket in 'us-east-1' (destination bucket). + err = c.MakeBucket(bucketName+"-copy", "us-east-1") + if err != nil { + logger().Fatal("Error:", err, bucketName+"-copy") + } + + // Generate data more than 32K + var buf = getDataBuffer("datafile-33-kB", rand.Intn(1<<20)+32*1024) + + // Save the data + objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") + n, err := c.PutObject(bucketName, objectName, bytes.NewReader(buf), "binary/octet-stream") + if err != nil { + logger().Fatal("Error:", err, bucketName, objectName) + } + + if n != int64(len(buf)) { + logger().Fatalf("Error: number of bytes does not match want %v, got %v", + len(buf), n) + } + + r, err := c.GetObject(bucketName, objectName) + if err != nil { + logger().Fatal("Error:", err) + } + // Check the various fields of source object against destination object. + objInfo, err := r.Stat() + if err != nil { + logger().Fatal("Error:", err) + } + + // Copy Source + src := minio.NewSourceInfo(bucketName, objectName, nil) + + // Set copy conditions. + + // All invalid conditions first. + err = src.SetModifiedSinceCond(time.Date(1, time.January, 1, 0, 0, 0, 0, time.UTC)) + if err == nil { + logger().Fatal("Error:", err) + } + err = src.SetUnmodifiedSinceCond(time.Date(1, time.January, 1, 0, 0, 0, 0, time.UTC)) + if err == nil { + logger().Fatal("Error:", err) + } + err = src.SetMatchETagCond("") + if err == nil { + logger().Fatal("Error:", err) + } + err = src.SetMatchETagExceptCond("") + if err == nil { + logger().Fatal("Error:", err) + } + + err = src.SetModifiedSinceCond(time.Date(2014, time.April, 0, 0, 0, 0, 0, time.UTC)) + if err != nil { + logger().Fatal("Error:", err) + } + err = src.SetMatchETagCond(objInfo.ETag) + if err != nil { + logger().Fatal("Error:", err) + } + + dst, err := minio.NewDestinationInfo(bucketName+"-copy", objectName+"-copy", nil, nil) + if err != nil { + logger().Fatal(err) + } + + // Perform the Copy + err = c.CopyObject(dst, src) + if err != nil { + logger().Fatal("Error:", err, bucketName+"-copy", objectName+"-copy") + } + + // Source object + reader, err := c.GetObject(bucketName, objectName) + if err != nil { + logger().Fatal("Error:", err) + } + // Destination object + readerCopy, err := c.GetObject(bucketName+"-copy", objectName+"-copy") + if err != nil { + logger().Fatal("Error:", err) + } + // Check the various fields of source object against destination object. + objInfo, err = reader.Stat() + if err != nil { + logger().Fatal("Error:", err) + } + objInfoCopy, err := readerCopy.Stat() + if err != nil { + logger().Fatal("Error:", err) + } + if objInfo.Size != objInfoCopy.Size { + logger().Fatalf("Error: number of bytes does not match, want %v, got %v\n", + objInfo.Size, objInfoCopy.Size) + } + + // CopyObject again but with wrong conditions + src = minio.NewSourceInfo(bucketName, objectName, nil) + err = src.SetUnmodifiedSinceCond(time.Date(2014, time.April, 0, 0, 0, 0, 0, time.UTC)) + if err != nil { + logger().Fatal("Error:", err) + } + err = src.SetMatchETagExceptCond(objInfo.ETag) + if err != nil { + logger().Fatal("Error:", err) + } + + // Perform the Copy which should fail + err = c.CopyObject(dst, src) + if err == nil { + logger().Fatal("Error:", err, bucketName+"-copy", objectName+"-copy should fail") + } + + // Remove all objects and buckets + err = c.RemoveObject(bucketName, objectName) + if err != nil { + logger().Fatal("Error:", err) + } + + err = c.RemoveObject(bucketName+"-copy", objectName+"-copy") + if err != nil { + logger().Fatal("Error:", err) + } + + err = c.RemoveBucket(bucketName) + if err != nil { + logger().Fatal("Error:", err) + } + + err = c.RemoveBucket(bucketName + "-copy") + if err != nil { + logger().Fatal("Error:", err) + } +} + +func testComposeObjectErrorCasesWrapper(c *minio.Client) { + // Generate a new random bucket name. + bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") + + // Make a new bucket in 'us-east-1' (source bucket). + err := c.MakeBucket(bucketName, "us-east-1") + if err != nil { + logger().Fatal("Error:", err, bucketName) + } + + // Test that more than 10K source objects cannot be + // concatenated. + srcArr := [10001]minio.SourceInfo{} + srcSlice := srcArr[:] + dst, err := minio.NewDestinationInfo(bucketName, "object", nil, nil) + if err != nil { + logger().Fatal(err) + } + + if err := c.ComposeObject(dst, srcSlice); err == nil { + logger().Fatal("Error was expected.") + } else if err.Error() != "There must be as least one and upto 10000 source objects." { + logger().Fatal("Got unexpected error: ", err) + } + + // Create a source with invalid offset spec and check that + // error is returned: + // 1. Create the source object. + const badSrcSize = 5 * 1024 * 1024 + buf := bytes.Repeat([]byte("1"), badSrcSize) + _, err = c.PutObject(bucketName, "badObject", bytes.NewReader(buf), "") + if err != nil { + logger().Fatal("Error:", err) + } + // 2. Set invalid range spec on the object (going beyond + // object size) + badSrc := minio.NewSourceInfo(bucketName, "badObject", nil) + err = badSrc.SetRange(1, badSrcSize) + if err != nil { + logger().Fatal("Error:", err) + } + // 3. ComposeObject call should fail. + if err := c.ComposeObject(dst, []minio.SourceInfo{badSrc}); err == nil { + logger().Fatal("Error was expected.") + } else if !strings.Contains(err.Error(), "has invalid segment-to-copy") { + logger().Fatal("Got unexpected error: ", err) + } +} + +// Test expected error cases +func testComposeObjectErrorCasesV2() { + logger().Info() + + // Instantiate new minio client object + c, err := minio.NewV2( + os.Getenv(serverEndpoint), + os.Getenv(accessKey), + os.Getenv(secretKey), + mustParseBool(os.Getenv(enableHTTPS)), + ) + if err != nil { + logger().Fatal("Error:", err) + } + + testComposeObjectErrorCasesWrapper(c) +} + +func testComposeMultipleSources(c *minio.Client) { + // Generate a new random bucket name. + bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") + // Make a new bucket in 'us-east-1' (source bucket). + err := c.MakeBucket(bucketName, "us-east-1") + if err != nil { + logger().Fatal("Error:", err, bucketName) + } + + // Upload a small source object + const srcSize = 1024 * 1024 * 5 + buf := bytes.Repeat([]byte("1"), srcSize) + _, err = c.PutObject(bucketName, "srcObject", bytes.NewReader(buf), "binary/octet-stream") + if err != nil { + logger().Fatal("Error:", err) + } + + // We will append 10 copies of the object. + srcs := []minio.SourceInfo{} + for i := 0; i < 10; i++ { + srcs = append(srcs, minio.NewSourceInfo(bucketName, "srcObject", nil)) + } + // make the last part very small + err = srcs[9].SetRange(0, 0) + if err != nil { + logger().Fatal("unexpected error:", err) + } + + dst, err := minio.NewDestinationInfo(bucketName, "dstObject", nil, nil) + if err != nil { + logger().Fatal(err) + } + err = c.ComposeObject(dst, srcs) + if err != nil { + logger().Fatal("Error:", err) + } + + objProps, err := c.StatObject(bucketName, "dstObject") + if err != nil { + logger().Fatal("Error:", err) + } + + if objProps.Size != 9*srcSize+1 { + logger().Fatal("Size mismatched! Expected:", 10000*srcSize, "but got:", objProps.Size) + } +} + +// Test concatenating multiple objects objects +func testCompose10KSourcesV2() { + logger().Info() + + // Instantiate new minio client object + c, err := minio.NewV2( + os.Getenv(serverEndpoint), + os.Getenv(accessKey), + os.Getenv(secretKey), + mustParseBool(os.Getenv(enableHTTPS)), + ) + if err != nil { + logger().Fatal("Error:", err) + } + + testComposeMultipleSources(c) +} +func testEncryptedCopyObjectWrapper(c *minio.Client) { + // Generate a new random bucket name. + bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") + // Make a new bucket in 'us-east-1' (source bucket). + err := c.MakeBucket(bucketName, "us-east-1") + if err != nil { + logger().Fatal("Error:", err, bucketName) + } + + key1 := minio.NewSSEInfo([]byte("32byteslongsecretkeymustbegiven1"), "AES256") + key2 := minio.NewSSEInfo([]byte("32byteslongsecretkeymustbegiven2"), "AES256") + + // 1. create an sse-c encrypted object to copy by uploading + const srcSize = 1024 * 1024 + buf := bytes.Repeat([]byte("abcde"), srcSize) // gives a buffer of 5MiB + metadata := make(map[string][]string) + for k, v := range key1.GetSSEHeaders() { + metadata[k] = append(metadata[k], v) + } + _, err = c.PutObjectWithSize(bucketName, "srcObject", bytes.NewReader(buf), int64(len(buf)), metadata, nil) + if err != nil { + logger().Fatal("PutObjectWithSize Error:", err) + } + + // 2. copy object and change encryption key + src := minio.NewSourceInfo(bucketName, "srcObject", &key1) + dst, err := minio.NewDestinationInfo(bucketName, "dstObject", &key2, nil) + if err != nil { + logger().Fatal("Error:", err) + } + + err = c.CopyObject(dst, src) + if err != nil { + logger().Fatal("CopyObject Error:", err) + } + + // 3. get copied object and check if content is equal + reqH := minio.NewGetReqHeaders() + for k, v := range key2.GetSSEHeaders() { + reqH.Set(k, v) + } + coreClient := minio.Core{c} + reader, _, err := coreClient.GetObject(bucketName, "dstObject", reqH) + if err != nil { + logger().Fatal("GetObject Error:", err) + } + defer reader.Close() + + decBytes, err := ioutil.ReadAll(reader) + if err != nil { + logger().Fatalln(err) + } + if !bytes.Equal(decBytes, buf) { + logger().Fatal("downloaded object mismatched for encrypted object") + } +} + +// Test encrypted copy object +func testEncryptedCopyObject() { + logger().Info() + + // Instantiate new minio client object + c, err := minio.NewV4( + os.Getenv(serverEndpoint), + os.Getenv(accessKey), + os.Getenv(secretKey), + mustParseBool(os.Getenv(enableHTTPS)), + ) + if err != nil { + logger().Fatal("Error:", err) + } + + // c.TraceOn(os.Stderr) + testEncryptedCopyObjectWrapper(c) +} + +// Test encrypted copy object +func testEncryptedCopyObjectV2() { + logger().Info() + + // Instantiate new minio client object + c, err := minio.NewV2( + os.Getenv(serverEndpoint), + os.Getenv(accessKey), + os.Getenv(secretKey), + mustParseBool(os.Getenv(enableHTTPS)), + ) + if err != nil { + logger().Fatal("Error:", err) + } + + testEncryptedCopyObjectWrapper(c) +} +func testUserMetadataCopying() { + logger().Info() + + // Instantiate new minio client object + c, err := minio.NewV4( + os.Getenv(serverEndpoint), + os.Getenv(accessKey), + os.Getenv(secretKey), + mustParseBool(os.Getenv(enableHTTPS)), + ) + if err != nil { + logger().Fatal("Error:", err) + } + + // c.TraceOn(os.Stderr) + testUserMetadataCopyingWrapper(c) +} +func testUserMetadataCopyingWrapper(c *minio.Client) { + // Generate a new random bucket name. + bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") + // Make a new bucket in 'us-east-1' (source bucket). + err := c.MakeBucket(bucketName, "us-east-1") + if err != nil { + logger().Fatal("Error:", err, bucketName) + } + + fetchMeta := func(object string) (h http.Header) { + objInfo, err := c.StatObject(bucketName, object) + if err != nil { + logger().Fatal("Metadata fetch error:", err) + } + h = make(http.Header) + for k, vs := range objInfo.Metadata { + if strings.HasPrefix(strings.ToLower(k), "x-amz-meta-") { + for _, v := range vs { + h.Add(k, v) + } + } + } + return h + } + + // 1. create a client encrypted object to copy by uploading + const srcSize = 1024 * 1024 + buf := bytes.Repeat([]byte("abcde"), srcSize) // gives a buffer of 5MiB + metadata := make(http.Header) + metadata.Set("x-amz-meta-myheader", "myvalue") + _, err = c.PutObjectWithMetadata(bucketName, "srcObject", + bytes.NewReader(buf), metadata, nil) + if err != nil { + logger().Fatal("Put Error:", err) + } + if !reflect.DeepEqual(metadata, fetchMeta("srcObject")) { + logger().Fatal("Unequal metadata") + } + + // 2. create source + src := minio.NewSourceInfo(bucketName, "srcObject", nil) + // 2.1 create destination with metadata set + dst1, err := minio.NewDestinationInfo(bucketName, "dstObject-1", nil, map[string]string{"notmyheader": "notmyvalue"}) + if err != nil { + logger().Fatal("Error:", err) + } + + // 3. Check that copying to an object with metadata set resets + // the headers on the copy. + err = c.CopyObject(dst1, src) + if err != nil { + logger().Fatal("Error:", err) + } + + expectedHeaders := make(http.Header) + expectedHeaders.Set("x-amz-meta-notmyheader", "notmyvalue") + if !reflect.DeepEqual(expectedHeaders, fetchMeta("dstObject-1")) { + logger().Fatal("Unequal metadata") + } + + // 4. create destination with no metadata set and same source + dst2, err := minio.NewDestinationInfo(bucketName, "dstObject-2", nil, nil) + if err != nil { + logger().Fatal("Error:", err) + + } + src = minio.NewSourceInfo(bucketName, "srcObject", nil) + + // 5. Check that copying to an object with no metadata set, + // copies metadata. + err = c.CopyObject(dst2, src) + if err != nil { + logger().Fatal("Error:", err) + } + + expectedHeaders = metadata + if !reflect.DeepEqual(expectedHeaders, fetchMeta("dstObject-2")) { + logger().Fatal("Unequal metadata") + } + + // 6. Compose a pair of sources. + srcs := []minio.SourceInfo{ + minio.NewSourceInfo(bucketName, "srcObject", nil), + minio.NewSourceInfo(bucketName, "srcObject", nil), + } + dst3, err := minio.NewDestinationInfo(bucketName, "dstObject-3", nil, nil) + if err != nil { + logger().Fatal("Error:", err) + + } + + err = c.ComposeObject(dst3, srcs) + if err != nil { + logger().Fatal("Error:", err) + } + + // Check that no headers are copied in this case + if !reflect.DeepEqual(make(http.Header), fetchMeta("dstObject-3")) { + logger().Fatal("Unequal metadata") + } + + // 7. Compose a pair of sources with dest user metadata set. + srcs = []minio.SourceInfo{ + minio.NewSourceInfo(bucketName, "srcObject", nil), + minio.NewSourceInfo(bucketName, "srcObject", nil), + } + dst4, err := minio.NewDestinationInfo(bucketName, "dstObject-4", nil, map[string]string{"notmyheader": "notmyvalue"}) + if err != nil { + logger().Fatal("Error:", err) + + } + + err = c.ComposeObject(dst4, srcs) + if err != nil { + logger().Fatal("Error:", err) + } + + // Check that no headers are copied in this case + expectedHeaders = make(http.Header) + expectedHeaders.Set("x-amz-meta-notmyheader", "notmyvalue") + if !reflect.DeepEqual(expectedHeaders, fetchMeta("dstObject-4")) { + logger().Fatal("Unequal metadata") + } +} + +func testUserMetadataCopyingV2() { + logger().Info() + + // Instantiate new minio client object + c, err := minio.NewV2( + os.Getenv(serverEndpoint), + os.Getenv(accessKey), + os.Getenv(secretKey), + mustParseBool(os.Getenv(enableHTTPS)), + ) + if err != nil { + logger().Fatal("Error:", err) + } + + // c.TraceOn(os.Stderr) + testUserMetadataCopyingWrapper(c) +} + +// Test put object with 0 byte object. +func testPutObject0ByteV2() { + logTrace() + + // Seed random based on current time. + rand.Seed(time.Now().Unix()) + + // Instantiate new minio client object. + c, err := minio.NewV2( + os.Getenv(serverEndpoint), + os.Getenv(accessKey), + os.Getenv(secretKey), + mustParseBool(os.Getenv(enableHTTPS)), + ) + if err != nil { + log.Fatal("Error:", err) + } + + // Enable tracing, write to stderr. + // c.TraceOn(os.Stderr) + + // Set user agent. + c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + + // Generate a new random bucket name. + bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), + "minio-go-test") + + // Make a new bucket. + err = c.MakeBucket(bucketName, "us-east-1") + if err != nil { + log.Fatal("Error:", err, bucketName) + } + + objectName := bucketName + "unique" + + // Upload an object. + n, err := c.PutObjectWithSize(bucketName, objectName, bytes.NewReader([]byte("")), 0, nil, nil) + if err != nil { + log.Fatalf("Error: %v %s %s", err, bucketName, objectName) + } + if n != 0 { + log.Error(fmt.Errorf("Expected upload object size 0 but got %d", n)) + } + + // Remove the object. + err = c.RemoveObject(bucketName, objectName) + if err != nil { + log.Fatal("Error:", err) + } + + // Remove the bucket. + err = c.RemoveBucket(bucketName) + if err != nil { + log.Fatal("Error:", err) + } +} + +// Test expected error cases +func testComposeObjectErrorCases() { + logger().Info() + + // Instantiate new minio client object + c, err := minio.NewV4( + os.Getenv(serverEndpoint), + os.Getenv(accessKey), + os.Getenv(secretKey), + mustParseBool(os.Getenv(enableHTTPS)), + ) + if err != nil { + logger().Fatal("Error:", err) + } + + testComposeObjectErrorCasesWrapper(c) +} + +// Test concatenating 10K objects +func testCompose10KSources() { + logger().Info() + + // Instantiate new minio client object + c, err := minio.NewV4( + os.Getenv(serverEndpoint), + os.Getenv(accessKey), + os.Getenv(secretKey), + mustParseBool(os.Getenv(enableHTTPS)), + ) + if err != nil { + logger().Fatal("Error:", err) + } + + testComposeMultipleSources(c) +} + +// Tests comprehensive list of all methods. +func testFunctionalV2() { + logger().Info() + // Seed random based on current time. + rand.Seed(time.Now().Unix()) + + c, err := minio.NewV2( + os.Getenv(serverEndpoint), + os.Getenv(accessKey), + os.Getenv(secretKey), + mustParseBool(os.Getenv(enableHTTPS)), + ) + if err != nil { + logger().Fatal("Error:", err) + } + + // Enable to debug + // c.TraceOn(os.Stderr) + + // Set user agent. + c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + + // Generate a new random bucket name. + bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") + + // Make a new bucket. + err = c.MakeBucket(bucketName, "us-east-1") + if err != nil { + logger().Fatal("Error:", err, bucketName) + } + + // Generate a random file name. + fileName := randString(60, rand.NewSource(time.Now().UnixNano()), "") + file, err := os.Create(fileName) + if err != nil { + logger().Fatal("Error:", err) + } + for i := 0; i < 3; i++ { + buf := make([]byte, rand.Intn(1<<19)) + _, err = file.Write(buf) + if err != nil { + logger().Fatal("Error:", err) + } + } + file.Close() + + // Verify if bucket exits and you have access. + var exists bool + exists, err = c.BucketExists(bucketName) + if err != nil { + logger().Fatal("Error:", err, bucketName) + } + if !exists { + logger().Fatal("Error: could not find ", bucketName) + } + + // Make the bucket 'public read/write'. + err = c.SetBucketPolicy(bucketName, "", policy.BucketPolicyReadWrite) + if err != nil { + logger().Fatal("Error:", err) + } + + // List all buckets. + buckets, err := c.ListBuckets() + if len(buckets) == 0 { + logger().Fatal("Error: list buckets cannot be empty", buckets) + } + if err != nil { + logger().Fatal("Error:", err) + } + + // Verify if previously created bucket is listed in list buckets. + bucketFound := false + for _, bucket := range buckets { + if bucket.Name == bucketName { + bucketFound = true + } + } + + // If bucket not found error out. + if !bucketFound { + logger().Fatal("Error: bucket ", bucketName, "not found") + } + + objectName := bucketName + "unique" + + // Generate data + buf := bytes.Repeat([]byte("n"), rand.Intn(1<<19)) + + n, err := c.PutObject(bucketName, objectName, bytes.NewReader(buf), "") + if err != nil { + logger().Fatal("Error: ", err) + } + if n != int64(len(buf)) { + logger().Fatal("Error: bad length ", n, len(buf)) + } + + n, err = c.PutObject(bucketName, objectName+"-nolength", bytes.NewReader(buf), "binary/octet-stream") + if err != nil { + logger().Fatal("Error:", err, bucketName, objectName+"-nolength") + } + + if n != int64(len(buf)) { + logger().Fatalf("Error: number of bytes does not match, want %v, got %v\n", len(buf), n) + } + + // Instantiate a done channel to close all listing. + doneCh := make(chan struct{}) + defer close(doneCh) + + objFound := false + isRecursive := true // Recursive is true. + for obj := range c.ListObjects(bucketName, objectName, isRecursive, doneCh) { + if obj.Key == objectName { + objFound = true + break + } + } + if !objFound { + logger().Fatal("Error: object " + objectName + " not found.") + } + + objFound = false + isRecursive = true // Recursive is true. + for obj := range c.ListObjects(bucketName, objectName, isRecursive, doneCh) { + if obj.Key == objectName { + objFound = true + break + } + } + if !objFound { + logger().Fatal("Error: object " + objectName + " not found.") + } + + incompObjNotFound := true + for objIncompl := range c.ListIncompleteUploads(bucketName, objectName, isRecursive, doneCh) { + if objIncompl.Key != "" { + incompObjNotFound = false + break + } + } + if !incompObjNotFound { + logger().Fatal("Error: unexpected dangling incomplete upload found.") + } + + newReader, err := c.GetObject(bucketName, objectName) + if err != nil { + logger().Fatal("Error: ", err) + } + + newReadBytes, err := ioutil.ReadAll(newReader) + if err != nil { + logger().Fatal("Error: ", err) + } + + if !bytes.Equal(newReadBytes, buf) { + logger().Fatal("Error: bytes mismatch.") + } + + err = c.FGetObject(bucketName, objectName, fileName+"-f") + if err != nil { + logger().Fatal("Error: ", err) + } + + // Generate presigned GET object url. + presignedGetURL, err := c.PresignedGetObject(bucketName, objectName, 3600*time.Second, nil) + if err != nil { + logger().Fatal("Error: ", err) + } + // Verify if presigned url works. + resp, err := http.Get(presignedGetURL.String()) + if err != nil { + logger().Fatal("Error: ", err) + } + if resp.StatusCode != http.StatusOK { + logger().Fatal("Error: ", resp.Status) + } + newPresignedBytes, err := ioutil.ReadAll(resp.Body) + if err != nil { + logger().Fatal("Error: ", err) + } + if !bytes.Equal(newPresignedBytes, buf) { + logger().Fatal("Error: bytes mismatch.") + } + + // Set request parameters. + reqParams := make(url.Values) + reqParams.Set("response-content-disposition", "attachment; filename=\"test.txt\"") + // Generate presigned GET object url. + presignedGetURL, err = c.PresignedGetObject(bucketName, objectName, 3600*time.Second, reqParams) + if err != nil { + logger().Fatal("Error: ", err) + } + // Verify if presigned url works. + resp, err = http.Get(presignedGetURL.String()) + if err != nil { + logger().Fatal("Error: ", err) + } + if resp.StatusCode != http.StatusOK { + logger().Fatal("Error: ", resp.Status) + } + newPresignedBytes, err = ioutil.ReadAll(resp.Body) + if err != nil { + logger().Fatal("Error: ", err) + } + if !bytes.Equal(newPresignedBytes, buf) { + logger().Fatal("Error: bytes mismatch for presigned GET url.") + } + // Verify content disposition. + if resp.Header.Get("Content-Disposition") != "attachment; filename=\"test.txt\"" { + logger().Fatalf("Error: wrong Content-Disposition received %s", resp.Header.Get("Content-Disposition")) + } + + presignedPutURL, err := c.PresignedPutObject(bucketName, objectName+"-presigned", 3600*time.Second) + if err != nil { + logger().Fatal("Error: ", err) + } + // Generate data more than 32K + buf = bytes.Repeat([]byte("1"), rand.Intn(1<<20)+32*1024) + + req, err := http.NewRequest("PUT", presignedPutURL.String(), bytes.NewReader(buf)) + if err != nil { + logger().Fatal("Error: ", err) + } + httpClient := &http.Client{ + // Setting a sensible time out of 30secs to wait for response + // headers. Request is pro-actively cancelled after 30secs + // with no response. + Timeout: 30 * time.Second, + Transport: http.DefaultTransport, + } + resp, err = httpClient.Do(req) + if err != nil { + logger().Fatal("Error: ", err) + } + + newReader, err = c.GetObject(bucketName, objectName+"-presigned") + if err != nil { + logger().Fatal("Error: ", err) + } + + newReadBytes, err = ioutil.ReadAll(newReader) + if err != nil { + logger().Fatal("Error: ", err) + } + + if !bytes.Equal(newReadBytes, buf) { + logger().Fatal("Error: bytes mismatch.") + } + + err = c.RemoveObject(bucketName, objectName) + if err != nil { + logger().Fatal("Error: ", err) + } + err = c.RemoveObject(bucketName, objectName+"-f") + if err != nil { + logger().Fatal("Error: ", err) + } + err = c.RemoveObject(bucketName, objectName+"-nolength") + if err != nil { + logger().Fatal("Error: ", err) + } + err = c.RemoveObject(bucketName, objectName+"-presigned") + if err != nil { + logger().Fatal("Error: ", err) + } + err = c.RemoveBucket(bucketName) + if err != nil { + logger().Fatal("Error:", err) + } + err = c.RemoveBucket(bucketName) + if err == nil { + logger().Fatal("Error:") + } + if err.Error() != "The specified bucket does not exist" { + logger().Fatal("Error: ", err) + } + if err = os.Remove(fileName); err != nil { + logger().Fatal("Error: ", err) + } + if err = os.Remove(fileName + "-f"); err != nil { + logger().Fatal("Error: ", err) + } +} + +// Convert string to bool and always return false if any error +func mustParseBool(str string) bool { + b, err := strconv.ParseBool(str) + if err != nil { + return false + } + return b +} + +func logger() *logrus.Entry { + if pc, file, line, ok := runtime.Caller(1); ok { + fName := runtime.FuncForPC(pc).Name() + return log.WithFields(log.Fields{"file": path.Base(file), "function:": fName, "line#": line}) + + } + return log.WithFields(nil) +} + +func main() { + logger().Info("Running functional tests for minio-go sdk....") + if !isQuickMode() { + testMakeBucketErrorV2() + testGetObjectClosedTwiceV2() + testRemovePartiallyUploadedV2() + testFPutObjectV2() + testMakeBucketRegionsV2() + testGetObjectReadSeekFunctionalV2() + testGetObjectReadAtFunctionalV2() + testCopyObjectV2() + testFunctionalV2() + testComposeObjectErrorCasesV2() + testCompose10KSourcesV2() + testEncryptedCopyObjectV2() + testUserMetadataCopyingV2() + testPutObject0ByteV2() + testMakeBucketError() + testMakeBucketRegions() + testPutObjectWithMetadata() + testPutObjectReadAt() + testPutObjectStreaming() + testListPartiallyUploaded() + testGetObjectSeekEnd() + testGetObjectClosedTwice() + testRemoveMultipleObjects() + testRemovePartiallyUploaded() + testFPutObjectMultipart() + testFPutObject() + testGetObjectReadSeekFunctional() + testGetObjectReadAtFunctional() + testPresignedPostPolicy() + testCopyObject() + testEncryptionPutGet() + testComposeObjectErrorCases() + testCompose10KSources() + testUserMetadataCopying() + testEncryptedCopyObject() + testBucketNotification() + testFunctional() + testGetObjectObjectModified() + testPutObjectUploadSeekedObject() + } else { + logger().Info("Running short functional tests") + testFunctional() + testFunctionalV2() + } + + logger().Info("Functional tests complete for minio-go sdk") +} diff --git a/vendor/src/github.com/minio/minio-go/pkg/s3signer/request-signature-streaming.go b/vendor/src/github.com/minio/minio-go/pkg/s3signer/request-signature-streaming.go index 4fe010018..d831436cd 100644 --- a/vendor/src/github.com/minio/minio-go/pkg/s3signer/request-signature-streaming.go +++ b/vendor/src/github.com/minio/minio-go/pkg/s3signer/request-signature-streaming.go @@ -99,7 +99,7 @@ func prepareStreamingRequest(req *http.Request, sessionToken string, dataLen int if sessionToken != "" { req.Header.Set("X-Amz-Security-Token", sessionToken) } - req.Header.Set("Content-Encoding", streamingEncoding) + req.Header.Add("Content-Encoding", streamingEncoding) req.Header.Set("X-Amz-Date", timestamp.Format(iso8601DateFormat)) // Set content length with streaming signature for each chunk included. @@ -254,7 +254,18 @@ func (s *StreamingReader) Read(buf []byte) (int, error) { s.chunkBufLen = 0 for { n1, err := s.baseReadCloser.Read(s.chunkBuf[s.chunkBufLen:]) - if err == nil || err == io.ErrUnexpectedEOF { + // Usually we validate `err` first, but in this case + // we are validating n > 0 for the following reasons. + // + // 1. n > 0, err is one of io.EOF, nil (near end of stream) + // A Reader returning a non-zero number of bytes at the end + // of the input stream may return either err == EOF or err == nil + // + // 2. n == 0, err is io.EOF (actual end of stream) + // + // Callers should always process the n > 0 bytes returned + // before considering the error err. + if n1 > 0 { s.chunkBufLen += n1 s.bytesRead += int64(n1) @@ -265,25 +276,26 @@ func (s *StreamingReader) Read(buf []byte) (int, error) { s.signChunk(s.chunkBufLen) break } + } + if err != nil { + if err == io.EOF { + // No more data left in baseReader - last chunk. + // Done reading the last chunk from baseReader. + s.done = true - } else if err == io.EOF { - // No more data left in baseReader - last chunk. - // Done reading the last chunk from baseReader. - s.done = true + // bytes read from baseReader different than + // content length provided. + if s.bytesRead != s.contentLen { + return 0, io.ErrUnexpectedEOF + } - // bytes read from baseReader different than - // content length provided. - if s.bytesRead != s.contentLen { - return 0, io.ErrUnexpectedEOF + // Sign the chunk and write it to s.buf. + s.signChunk(0) + break } - - // Sign the chunk and write it to s.buf. - s.signChunk(0) - break - - } else { return 0, err } + } } return s.buf.Read(buf) diff --git a/vendor/src/github.com/minio/minio-go/tempfile.go b/vendor/src/github.com/minio/minio-go/tempfile.go deleted file mode 100644 index 65c7b0da1..000000000 --- a/vendor/src/github.com/minio/minio-go/tempfile.go +++ /dev/null @@ -1,60 +0,0 @@ -/* - * Minio Go Library for Amazon S3 Compatible Cloud Storage (C) 2015 Minio, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package minio - -import ( - "io/ioutil" - "os" - "sync" -) - -// tempFile - temporary file container. -type tempFile struct { - *os.File - mutex *sync.Mutex -} - -// newTempFile returns a new temporary file, once closed it automatically deletes itself. -func newTempFile(prefix string) (*tempFile, error) { - // use platform specific temp directory. - file, err := ioutil.TempFile(os.TempDir(), prefix) - if err != nil { - return nil, err - } - return &tempFile{ - File: file, - mutex: &sync.Mutex{}, - }, nil -} - -// Close - closer wrapper to close and remove temporary file. -func (t *tempFile) Close() error { - t.mutex.Lock() - defer t.mutex.Unlock() - if t.File != nil { - // Close the file. - if err := t.File.Close(); err != nil { - return err - } - // Remove file. - if err := os.Remove(t.File.Name()); err != nil { - return err - } - t.File = nil - } - return nil -} diff --git a/vendor/src/github.com/minio/minio-go/test-utils_test.go b/vendor/src/github.com/minio/minio-go/test-utils_test.go index 4134af996..b109dfaf7 100644 --- a/vendor/src/github.com/minio/minio-go/test-utils_test.go +++ b/vendor/src/github.com/minio/minio-go/test-utils_test.go @@ -64,11 +64,11 @@ func encodeResponse(response interface{}) []byte { return bytesBuffer.Bytes() } -// Convert string to bool and always return true if any error +// Convert string to bool and always return false if any error func mustParseBool(str string) bool { b, err := strconv.ParseBool(str) if err != nil { - return true + return false } return b } diff --git a/vendor/src/github.com/minio/minio-go/transport.go b/vendor/src/github.com/minio/minio-go/transport.go new file mode 100644 index 000000000..d286bd7ae --- /dev/null +++ b/vendor/src/github.com/minio/minio-go/transport.go @@ -0,0 +1,48 @@ +// +build go1.7 go1.8 + +/* + * Minio Go Library for Amazon S3 Compatible Cloud Storage + * (C) 2017 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package minio + +import ( + "net" + "net/http" + "time" +) + +// This default transport is similar to http.DefaultTransport +// but with additional DisableCompression: +var defaultMinioTransport http.RoundTripper = &http.Transport{ + Proxy: http.ProxyFromEnvironment, + DialContext: (&net.Dialer{ + Timeout: 30 * time.Second, + KeepAlive: 30 * time.Second, + DualStack: true, + }).DialContext, + MaxIdleConns: 100, + IdleConnTimeout: 90 * time.Second, + TLSHandshakeTimeout: 10 * time.Second, + ExpectContinueTimeout: 1 * time.Second, + // Set this value so that the underlying transport round-tripper + // doesn't try to auto decode the body of objects with + // content-encoding set to `gzip`. + // + // Refer: + // https://golang.org/src/net/http/transport.go?h=roundTrip#L1843 + DisableCompression: true, +} diff --git a/vendor/src/github.com/minio/minio-go/transport_1_5.go b/vendor/src/github.com/minio/minio-go/transport_1_5.go new file mode 100644 index 000000000..468daafd3 --- /dev/null +++ b/vendor/src/github.com/minio/minio-go/transport_1_5.go @@ -0,0 +1,39 @@ +// +build go1.5,!go1.6,!go1.7,!go1.8 + +/* + * Minio Go Library for Amazon S3 Compatible Cloud Storage + * (C) 2017 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package minio + +import ( + "net/http" + "time" +) + +// This default transport is similar to http.DefaultTransport +// but with additional DisableCompression: +var defaultMinioTransport http.RoundTripper = &http.Transport{ + Proxy: http.ProxyFromEnvironment, + TLSHandshakeTimeout: 10 * time.Second, + // Set this value so that the underlying transport round-tripper + // doesn't try to auto decode the body of objects with + // content-encoding set to `gzip`. + // + // Refer: + // https://golang.org/src/net/http/transport.go?h=roundTrip#L1843 + DisableCompression: true, +} diff --git a/vendor/src/github.com/minio/minio-go/transport_1_6.go b/vendor/src/github.com/minio/minio-go/transport_1_6.go new file mode 100644 index 000000000..77e7d76fc --- /dev/null +++ b/vendor/src/github.com/minio/minio-go/transport_1_6.go @@ -0,0 +1,40 @@ +// +build go1.6,!go1.7,!go1.8 + +/* + * Minio Go Library for Amazon S3 Compatible Cloud Storage + * (C) 2017 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package minio + +import ( + "net/http" + "time" +) + +// This default transport is similar to http.DefaultTransport +// but with additional DisableCompression: +var defaultMinioTransport http.RoundTripper = &http.Transport{ + Proxy: http.ProxyFromEnvironment, + TLSHandshakeTimeout: 10 * time.Second, + ExpectContinueTimeout: 1 * time.Second, + // Set this value so that the underlying transport round-tripper + // doesn't try to auto decode the body of objects with + // content-encoding set to `gzip`. + // + // Refer: + // https://golang.org/src/net/http/transport.go?h=roundTrip#L1843 + DisableCompression: true, +} diff --git a/vendor/src/github.com/minio/minio-go/utils.go b/vendor/src/github.com/minio/minio-go/utils.go index d7f0181e8..6f54639e0 100644 --- a/vendor/src/github.com/minio/minio-go/utils.go +++ b/vendor/src/github.com/minio/minio-go/utils.go @@ -122,7 +122,7 @@ func isValidEndpointURL(endpointURL url.URL) error { if endpointURL.Path != "/" && endpointURL.Path != "" { return ErrInvalidArgument("Endpoint url cannot have fully qualified paths.") } - if strings.Contains(endpointURL.Host, ".amazonaws.com") { + if strings.Contains(endpointURL.Host, ".s3.amazonaws.com") { if !s3utils.IsAmazonEndpoint(endpointURL) { return ErrInvalidArgument("Amazon S3 endpoint should be 's3.amazonaws.com'.") } diff --git a/vendor/src/github.com/minio/minio-go/utils_test.go b/vendor/src/github.com/minio/minio-go/utils_test.go index 11a6a8986..ba297112e 100644 --- a/vendor/src/github.com/minio/minio-go/utils_test.go +++ b/vendor/src/github.com/minio/minio-go/utils_test.go @@ -84,9 +84,9 @@ func TestGetEndpointURL(t *testing.T) { {"s3.cn-north-1.amazonaws.com.cn", false, "http://s3.cn-north-1.amazonaws.com.cn", nil, true}, {"192.168.1.1:9000", false, "http://192.168.1.1:9000", nil, true}, {"192.168.1.1:9000", true, "https://192.168.1.1:9000", nil, true}, + {"s3.amazonaws.com:443", true, "https://s3.amazonaws.com:443", nil, true}, {"13333.123123.-", true, "", ErrInvalidArgument(fmt.Sprintf("Endpoint: %s does not follow ip address or domain name standards.", "13333.123123.-")), false}, {"13333.123123.-", true, "", ErrInvalidArgument(fmt.Sprintf("Endpoint: %s does not follow ip address or domain name standards.", "13333.123123.-")), false}, - {"s3.amazonaws.com:443", true, "", ErrInvalidArgument("Amazon S3 endpoint should be 's3.amazonaws.com'."), false}, {"storage.googleapis.com:4000", true, "", ErrInvalidArgument("Google Cloud Storage endpoint should be 'storage.googleapis.com'."), false}, {"s3.aamzza.-", true, "", ErrInvalidArgument(fmt.Sprintf("Endpoint: %s does not follow ip address or domain name standards.", "s3.aamzza.-")), false}, {"", true, "", ErrInvalidArgument("Endpoint: does not follow ip address or domain name standards."), false}, @@ -132,10 +132,11 @@ func TestIsValidEndpointURL(t *testing.T) { {"https://s3-fips-us-gov-west-1.amazonaws.com", nil, true}, {"https://s3.amazonaws.com/", nil, true}, {"https://storage.googleapis.com/", nil, true}, + {"https://z3.amazonaws.com", nil, true}, + {"https://mybalancer.us-east-1.elb.amazonaws.com", nil, true}, {"192.168.1.1", ErrInvalidArgument("Endpoint url cannot have fully qualified paths."), false}, {"https://amazon.googleapis.com/", ErrInvalidArgument("Google Cloud Storage endpoint should be 'storage.googleapis.com'."), false}, {"https://storage.googleapis.com/bucket/", ErrInvalidArgument("Endpoint url cannot have fully qualified paths."), false}, - {"https://z3.amazonaws.com", ErrInvalidArgument("Amazon S3 endpoint should be 's3.amazonaws.com'."), false}, {"https://s3.amazonaws.com/bucket/object", ErrInvalidArgument("Endpoint url cannot have fully qualified paths."), false}, }