alexvanin/restic
1
0
Fork 0
forked from TrueCloudLab/restic
Code Pull requests Activity

Update minio-go

Browse source
This commit is contained in:
Alexander Neumann 2016-05-08 11:24:24 +02:00
parent 7dc7f0d295
commit 1b50d55d0c
13 changed files with 912 additions and 166 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
vendor/manifest vendored
View file

@ -28,7 +28,7 @@
{
"importpath": "github.com/minio/minio-go",
"repository": "https://github.com/minio/minio-go",
"revision": "17b4ebd52505bde655e3b14df732e31850641bb7",
"revision": "867b27701ad16db4a9f4dad40d28187ca8433ec9",
"branch": "master"
},
{

91
vendor/src/github.com/minio/minio-go/API.md vendored
View file

@ -60,10 +60,10 @@ s3Client can be used to perform operations on S3 storage. APIs are described bel
### Bucket operations
---------------------------------------
<a name="MakeBucket">
#### MakeBucket(bucketName, location)
#### MakeBucket(bucketName string, location string) error
Create a new bucket.
__Arguments__
__Parameters__
* `bucketName` _string_ - Name of the bucket.
* `location` _string_ - region valid values are _us-west-1_, _us-west-2_, _eu-west-1_, _eu-central-1_, _ap-southeast-1_, _ap-northeast-1_, _ap-southeast-2_, _sa-east-1_
@ -78,10 +78,10 @@ fmt.Println("Successfully created mybucket.")
```
---------------------------------------
<a name="ListBuckets">
#### ListBuckets()
List all buckets.
#### ListBuckets() ([]BucketInfo, error)
Lists all buckets.
`bucketList` emits bucket with the format:
`bucketList` lists bucket in the format:
* `bucket.Name` _string_: bucket name
* `bucket.CreationDate` time.Time : date when bucket was created
@ -98,10 +98,10 @@ for _, bucket := range buckets {
```
---------------------------------------
<a name="BucketExists">
#### BucketExists(bucketName)
#### BucketExists(bucketName string) error
Check if bucket exists.
__Arguments__
__Parameters__
* `bucketName` _string_ : name of the bucket
__Example__
@ -114,10 +114,10 @@ if err != nil {
```
---------------------------------------
<a name="RemoveBucket">
#### RemoveBucket(bucketName)
#### RemoveBucket(bucketName string) error
Remove a bucket.
__Arguments__
__Parameters__
* `bucketName` _string_ : name of the bucket
__Example__
@ -130,16 +130,16 @@ if err != nil {
```
---------------------------------------
<a name="GetBucketPolicy">
#### GetBucketPolicy(bucketName, objectPrefix)
#### GetBucketPolicy(bucketName string, objectPrefix string) error
Get access permissions on a bucket or a prefix.
__Arguments__
__Parameters__
* `bucketName` _string_ : name of the bucket
* `objectPrefix` _string_ : name of the object prefix
__Example__
```go
bucketPolicy, err := s3Client.GetBucketPolicy("mybucket")
bucketPolicy, err := s3Client.GetBucketPolicy("mybucket", "")
if err != nil {
fmt.Println(err)
return
@ -148,13 +148,13 @@ fmt.Println("Access permissions for mybucket is", bucketPolicy)
```
---------------------------------------
<a name="SetBucketPolicy">
#### SetBucketPolicy(bucketname, objectPrefix, policy)
#### SetBucketPolicy(bucketname string, objectPrefix string, policy BucketPolicy) error
Set access permissions on bucket or an object prefix.
__Arguments__
__Parameters__
* `bucketName` _string_: name of the bucket
* `objectPrefix` _string_ : name of the object prefix
* `policy` _BucketPolicy_: policy can be _non_, _readonly_, _readwrite_, _writeonly_
* `policy` _BucketPolicy_: policy can be _BucketPolicyNone_, _BucketPolicyReadOnly_, _BucketPolicyReadWrite_, _BucketPolicyWriteOnly_
__Example__
```go
@ -166,10 +166,10 @@ if err != nil {
```
---------------------------------------
<a name="RemoveBucketPolicy">
#### RemoveBucketPolicy(bucketname, objectPrefix)
#### RemoveBucketPolicy(bucketname string, objectPrefix string) error
Remove existing permissions on bucket or an object prefix.
__Arguments__
__Parameters__
* `bucketName` _string_: name of the bucket
* `objectPrefix` _string_ : name of the object prefix
@ -184,10 +184,10 @@ if err != nil {
---------------------------------------
<a name="ListObjects">
#### ListObjects(bucketName, prefix, recursive, doneCh)
#### ListObjects(bucketName string, prefix string, recursive bool, doneCh chan struct{}) <-chan ObjectInfo
List objects in a bucket.
__Arguments__
__Parameters__
* `bucketName` _string_: name of the bucket
* `objectPrefix` _string_: the prefix of the objects that should be listed
* `recursive` _bool_: `true` indicates recursive style listing and `false` indicates directory style listing delimited by '/'
@ -222,10 +222,10 @@ for object := range objectCh {
---------------------------------------
<a name="ListIncompleteUploads">
#### ListIncompleteUploads(bucketName, prefix, recursive)
#### ListIncompleteUploads(bucketName string, prefix string, recursive bool, doneCh chan struct{}) <-chan ObjectMultipartInfo
List partially uploaded objects in a bucket.
__Arguments__
__Parameters__
* `bucketname` _string_: name of the bucket
* `prefix` _string_: prefix of the object names that are partially uploaded
* `recursive` bool: directory style listing when false, recursive listing when true
@ -259,15 +259,15 @@ for multiPartObject := range multiPartObjectCh {
---------------------------------------
### Object operations
<a name="GetObject">
#### GetObject(bucketName, objectName)
#### GetObject(bucketName string, objectName string) *Object
Download an object.
__Arguments__
__Parameters__
* `bucketName` _string_: name of the bucket
* `objectName` _string_: name of the object
__Return Value__
* `object` _*minio.Object_ : _minio.Object_ represents object reader.
* `object` _*Object_ : _Object_ represents object reader.
__Example__
```go
@ -285,10 +285,10 @@ if _, err := io.Copy(localFile, object); err != nil {
---------------------------------------
---------------------------------------
<a name="FGetObject">
#### FGetObject(bucketName, objectName, filePath)
#### FGetObject(bucketName string, objectName string, filePath string) error
Callback is called with `error` in case of error or `null` in case of success
__Arguments__
__Parameters__
* `bucketName` _string_: name of the bucket
* `objectName` _string_: name of the object
* `filePath` _string_: path to which the object data will be written to
@ -303,11 +303,10 @@ if err != nil {
```
---------------------------------------
<a name="PutObject">
#### PutObject(bucketName, objectName, reader, contentType)
Upload an object.
#### PutObject(bucketName string, objectName string, reader io.Reader, contentType string) (n int, err error)
Upload contents from `io.Reader` to objectName.
Uploading a stream
__Arguments__
__Parameters__
* `bucketName` _string_: name of the bucket
* `objectName` _string_: name of the object
* `reader` _io.Reader_: Any golang object implementing io.Reader
@ -331,10 +330,10 @@ if err != nil {
---------------------------------------
<a name="CopyObject">
#### CopyObject(bucketName, objectName, objectSource, conditions)
#### CopyObject(bucketName string, objectName string, objectSource string, conditions CopyConditions) error
Copy a source object into a new object with the provided name in the provided bucket.
__Arguments__
__Parameters__
* `bucketName` _string_: name of the bucket
* `objectName` _string_: name of the object
* `objectSource` _string_: name of the object source.
@ -367,10 +366,10 @@ if err != nil {
---------------------------------------
<a name="FPutObject">
#### FPutObject(bucketName, objectName, filePath, contentType)
#### FPutObject(bucketName string, objectName string, filePath string, contentType string) error
Uploads the object using contents from a file
__Arguments__
__Parameters__
* `bucketName` _string_: name of the bucket
* `objectName` _string_: name of the object
* `filePath` _string_: file path of the file to be uploaded
@ -386,10 +385,10 @@ if err != nil {
```
---------------------------------------
<a name="StatObject">
#### StatObject(bucketName, objectName)
#### StatObject(bucketName string, objectName string) (ObjectInfo, error)
Get metadata of an object.
__Arguments__
__Parameters__
* `bucketName` _string_: name of the bucket
* `objectName` _string_: name of the object
@ -411,10 +410,10 @@ fmt.Println(objInfo)
```
---------------------------------------
<a name="RemoveObject">
#### RemoveObject(bucketName, objectName)
#### RemoveObject(bucketName string, objectName string) error
Remove an object.
__Arguments__
__Parameters__
* `bucketName` _string_: name of the bucket
* `objectName` _string_: name of the object
@ -428,10 +427,10 @@ if err != nil {
```
---------------------------------------
<a name="RemoveIncompleteUpload">
#### RemoveIncompleteUpload(bucketName, objectName)
#### RemoveIncompleteUpload(bucketName string, objectName string) error
Remove an partially uploaded object.
__Arguments__
__Parameters__
* `bucketName` _string_: name of the bucket
* `objectName` _string_: name of the object
@ -447,14 +446,14 @@ if err != nil {
### Presigned operations
---------------------------------------
<a name="PresignedGetObject">
#### PresignedGetObject(bucketName, objectName, expiry)
#### PresignedGetObject(bucketName, objectName string, expiry time.Duration, reqParams url.Values) error
Generate a presigned URL for GET.
__Arguments__
__Parameters__
* `bucketName` _string_: name of the bucket.
* `objectName` _string_: name of the object.
* `expiry` _time.Duration_: expiry in seconds.
`reqParams` _url.Values_ : additional response header overrides supports _response-expires_, _response-content-type_, _response-cache-control_, _response-content-disposition_
* `reqParams` _url.Values_ : additional response header overrides supports _response-expires_, _response-content-type_, _response-cache-control_, _response-content-disposition_
__Example__
```go
@ -472,13 +471,13 @@ if err != nil {
---------------------------------------
<a name="PresignedPutObject">
#### PresignedPutObject(bucketName, objectName, expiry)
#### PresignedPutObject(bucketName string, objectName string, expiry time.Duration) (string, error)
Generate a presigned URL for PUT.
<blockquote>
NOTE: you can upload to S3 only with specified object name.
</blockquote>
__Arguments__
__Parameters__
* `bucketName` _string_: name of the bucket
* `objectName` _string_: name of the object
* `expiry` _time.Duration_: expiry in seconds
@ -495,7 +494,7 @@ if err != nil {
---------------------------------------
<a name="PresignedPostPolicy">
#### PresignedPostPolicy
#### PresignedPostPolicy(policy PostPolicy) (map[string]string, error)
PresignedPostPolicy we can provide policies specifying conditions restricting
what you want to allow in a POST request, such as bucket name where objects can be
uploaded, key name prefixes that you want to allow for the object being created and more.

277
vendor/src/github.com/minio/minio-go/api-error-response_test.go vendored Normal file
View file

@ -0,0 +1,277 @@
/*
* Minio Go Library for Amazon S3 Compatible Cloud Storage (C) 2015 Minio, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required bZy applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package minio
import (
"bytes"
"encoding/xml"
"fmt"
"io/ioutil"
"net/http"
"reflect"
"strconv"
"testing"
)
// Tests validate the Error generator function for http response with error.
func TestHttpRespToErrorResponse(t *testing.T) {
// 'genAPIErrorResponse' generates ErrorResponse for given APIError.
// provides a encodable populated response values.
genAPIErrorResponse := func(err APIError, bucketName string) ErrorResponse {
var errResp = ErrorResponse{}
errResp.Code = err.Code
errResp.Message = err.Description
errResp.BucketName = bucketName
return errResp
}
// Encodes the response headers into XML format.
encodeErr := func(response interface{}) []byte {
var bytesBuffer bytes.Buffer
bytesBuffer.WriteString(xml.Header)
encode := xml.NewEncoder(&bytesBuffer)
encode.Encode(response)
return bytesBuffer.Bytes()
}
// `createAPIErrorResponse` Mocks XML error response from the server.
createAPIErrorResponse := func(APIErr APIError, bucketName string) *http.Response {
// generate error response.
// response body contains the XML error message.
resp := &http.Response{}
errorResponse := genAPIErrorResponse(APIErr, bucketName)
encodedErrorResponse := encodeErr(errorResponse)
// write Header.
resp.StatusCode = APIErr.HTTPStatusCode
resp.Body = ioutil.NopCloser(bytes.NewBuffer(encodedErrorResponse))
return resp
}
// 'genErrResponse' contructs error response based http Status Code
genErrResponse := func(resp *http.Response, code, message, bucketName, objectName string) ErrorResponse {
errResp := ErrorResponse{
Code: code,
Message: message,
BucketName: bucketName,
Key: objectName,
RequestID: resp.Header.Get("x-amz-request-id"),
HostID: resp.Header.Get("x-amz-id-2"),
Region: resp.Header.Get("x-amz-bucket-region"),
}
return errResp
}
// Generate invalid argument error.
genInvalidError := func(message string) error {
errResp := ErrorResponse{
Code: "InvalidArgument",
Message: message,
RequestID: "minio",
}
return errResp
}
// Set common http response headers.
setCommonHeaders := func(resp *http.Response) *http.Response {
// set headers.
resp.Header = make(http.Header)
resp.Header.Set("x-amz-request-id", "xyz")
resp.Header.Set("x-amz-id-2", "abc")
resp.Header.Set("x-amz-bucket-region", "us-east-1")
return resp
}
// Generate http response with empty body.
// Set the StatusCode to the arugment supplied.
// Sets common headers.
genEmptyBodyResponse := func(statusCode int) *http.Response {
resp := &http.Response{}
// set empty response body.
resp.Body = ioutil.NopCloser(bytes.NewBuffer([]byte("")))
// set headers.
setCommonHeaders(resp)
// set status code.
resp.StatusCode = statusCode
return resp
}
// Decode XML error message from the http response body.
decodeXMLError := func(resp *http.Response, t *testing.T) error {
var errResp ErrorResponse
err := xmlDecoder(resp.Body, &errResp)
if err != nil {
t.Fatal("XML decoding of response body failed")
}
return errResp
}
// List of APIErrors used to generate/mock server side XML error response.
APIErrors := []APIError{
{
Code: "NoSuchBucketPolicy",
Description: "The specified bucket does not have a bucket policy.",
HTTPStatusCode: http.StatusNotFound,
},
}
// List of expected response.
// Used for asserting the actual response.
expectedErrResponse := []error{
genInvalidError("Response is empty. " + "Please report this issue at https://github.com/minio/minio-go/issues."),
decodeXMLError(createAPIErrorResponse(APIErrors[0], "minio-bucket"), t),
genErrResponse(setCommonHeaders(&http.Response{}), "NoSuchBucket", "The specified bucket does not exist.", "minio-bucket", ""),
genErrResponse(setCommonHeaders(&http.Response{}), "NoSuchKey", "The specified key does not exist.", "minio-bucket", "Asia/"),
genErrResponse(setCommonHeaders(&http.Response{}), "AccessDenied", "Access Denied.", "minio-bucket", ""),
genErrResponse(setCommonHeaders(&http.Response{}), "Conflict", "Bucket not empty.", "minio-bucket", ""),
genErrResponse(setCommonHeaders(&http.Response{}), "Bad Request", "Bad Request", "minio-bucket", ""),
}
// List of http response to be used as input.
inputResponses := []*http.Response{
nil,
createAPIErrorResponse(APIErrors[0], "minio-bucket"),
genEmptyBodyResponse(http.StatusNotFound),
genEmptyBodyResponse(http.StatusNotFound),
genEmptyBodyResponse(http.StatusForbidden),
genEmptyBodyResponse(http.StatusConflict),
genEmptyBodyResponse(http.StatusBadRequest),
}
testCases := []struct {
bucketName string
objectName string
inputHTTPResp *http.Response
// expected results.
expectedResult error
// flag indicating whether tests should pass.
}{
{"minio-bucket", "", inputResponses[0], expectedErrResponse[0]},
{"minio-bucket", "", inputResponses[1], expectedErrResponse[1]},
{"minio-bucket", "", inputResponses[2], expectedErrResponse[2]},
{"minio-bucket", "Asia/", inputResponses[3], expectedErrResponse[3]},
{"minio-bucket", "", inputResponses[4], expectedErrResponse[4]},
{"minio-bucket", "", inputResponses[5], expectedErrResponse[5]},
}
for i, testCase := range testCases {
actualResult := httpRespToErrorResponse(testCase.inputHTTPResp, testCase.bucketName, testCase.objectName)
if !reflect.DeepEqual(testCase.expectedResult, actualResult) {
t.Errorf("Test %d: Expected result to be '%+v', but instead got '%+v'", i+1, testCase.expectedResult, actualResult)
}
}
}
// Test validates 'ErrEntityTooLarge' error response.
func TestErrEntityTooLarge(t *testing.T) {
msg := fmt.Sprintf("Your proposed upload size ‘%d’ exceeds the maximum allowed object size ‘%d’ for single PUT operation.", 1000000, 99999)
expectedResult := ErrorResponse{
Code: "EntityTooLarge",
Message: msg,
BucketName: "minio-bucket",
Key: "Asia/",
}
actualResult := ErrEntityTooLarge(1000000, 99999, "minio-bucket", "Asia/")
if !reflect.DeepEqual(expectedResult, actualResult) {
t.Errorf("Expected result to be '%+v', but instead got '%+v'", expectedResult, actualResult)
}
}
// Test validates 'ErrEntityTooSmall' error response.
func TestErrEntityTooSmall(t *testing.T) {
msg := fmt.Sprintf("Your proposed upload size ‘%d’ is below the minimum allowed object size '0B' for single PUT operation.", -1)
expectedResult := ErrorResponse{
Code: "EntityTooLarge",
Message: msg,
BucketName: "minio-bucket",
Key: "Asia/",
}
actualResult := ErrEntityTooSmall(-1, "minio-bucket", "Asia/")
if !reflect.DeepEqual(expectedResult, actualResult) {
t.Errorf("Expected result to be '%+v', but instead got '%+v'", expectedResult, actualResult)
}
}
// Test validates 'ErrUnexpectedEOF' error response.
func TestErrUnexpectedEOF(t *testing.T) {
msg := fmt.Sprintf("Data read ‘%s’ is not equal to the size ‘%s’ of the input Reader.",
strconv.FormatInt(100, 10), strconv.FormatInt(101, 10))
expectedResult := ErrorResponse{
Code: "UnexpectedEOF",
Message: msg,
BucketName: "minio-bucket",
Key: "Asia/",
}
actualResult := ErrUnexpectedEOF(100, 101, "minio-bucket", "Asia/")
if !reflect.DeepEqual(expectedResult, actualResult) {
t.Errorf("Expected result to be '%+v', but instead got '%+v'", expectedResult, actualResult)
}
}
// Test validates 'ErrInvalidBucketName' error response.
func TestErrInvalidBucketName(t *testing.T) {
expectedResult := ErrorResponse{
Code: "InvalidBucketName",
Message: "Invalid Bucket name",
RequestID: "minio",
}
actualResult := ErrInvalidBucketName("Invalid Bucket name")
if !reflect.DeepEqual(expectedResult, actualResult) {
t.Errorf("Expected result to be '%+v', but instead got '%+v'", expectedResult, actualResult)
}
}
// Test validates 'ErrInvalidObjectName' error response.
func TestErrInvalidObjectName(t *testing.T) {
expectedResult := ErrorResponse{
Code: "NoSuchKey",
Message: "Invalid Object Key",
RequestID: "minio",
}
actualResult := ErrInvalidObjectName("Invalid Object Key")
if !reflect.DeepEqual(expectedResult, actualResult) {
t.Errorf("Expected result to be '%+v', but instead got '%+v'", expectedResult, actualResult)
}
}
// Test validates 'ErrInvalidParts' error response.
func TestErrInvalidParts(t *testing.T) {
msg := fmt.Sprintf("Unexpected number of parts found Want %d, Got %d", 10, 9)
expectedResult := ErrorResponse{
Code: "InvalidParts",
Message: msg,
RequestID: "minio",
}
actualResult := ErrInvalidParts(10, 9)
if !reflect.DeepEqual(expectedResult, actualResult) {
t.Errorf("Expected result to be '%+v', but instead got '%+v'", expectedResult, actualResult)
}
}
// Test validates 'ErrInvalidArgument' response.
func TestErrInvalidArgument(t *testing.T) {
expectedResult := ErrorResponse{
Code: "InvalidArgument",
Message: "Invalid Argument",
RequestID: "minio",
}
actualResult := ErrInvalidArgument("Invalid Argument")
if !reflect.DeepEqual(expectedResult, actualResult) {
t.Errorf("Expected result to be '%+v', but instead got '%+v'", expectedResult, actualResult)
}
}

2
vendor/src/github.com/minio/minio-go/api-get-policy.go vendored
View file

@ -61,7 +61,7 @@ func (c Client) getBucketPolicy(bucketName string, objectPrefix string) (BucketA
}
// processes the GetPolicy http resposne from the server.
// processes the GetPolicy http response from the server.
func processBucketPolicyResponse(bucketName string, resp *http.Response) (BucketAccessPolicy, error) {
if resp != nil {
if resp.StatusCode != http.StatusOK {

50
vendor/src/github.com/minio/minio-go/api-get-policy_test.go vendored
View file

@ -19,65 +19,24 @@ package minio
import (
"bytes"
"encoding/json"
"encoding/xml"
"io/ioutil"
"net/http"
"reflect"
"testing"
)
type APIError struct {
Code string
Description string
HTTPStatusCode int
}
// Mocks XML error response from the server.
func generateErrorResponse(resp *http.Response, APIErr APIError, bucketName string) *http.Response {
// generate error response.
errorResponse := getAPIErrorResponse(APIErr, bucketName)
encodedErrorResponse := encodeResponse(errorResponse)
// write Header.
resp.StatusCode = APIErr.HTTPStatusCode
resp.Body = ioutil.NopCloser(bytes.NewBuffer(encodedErrorResponse))
return resp
}
// getErrorResponse gets in standard error and resource value and
// provides a encodable populated response values.
func getAPIErrorResponse(err APIError, bucketName string) ErrorResponse {
var data = ErrorResponse{}
data.Code = err.Code
data.Message = err.Description
data.BucketName = bucketName
// TODO implement this in future
return data
}
// Encodes the response headers into XML format.
func encodeResponse(response interface{}) []byte {
var bytesBuffer bytes.Buffer
bytesBuffer.WriteString(xml.Header)
encode := xml.NewEncoder(&bytesBuffer)
encode.Encode(response)
return bytesBuffer.Bytes()
}
// Mocks valid http response containing bucket policy from server.
func generatePolicyResponse(resp *http.Response, policy BucketAccessPolicy) (*http.Response, error) {
policyBytes, e := json.Marshal(policy)
if e != nil {
return nil, e
policyBytes, err := json.Marshal(policy)
if err != nil {
return nil, err
}
resp.StatusCode = http.StatusOK
resp.Body = ioutil.NopCloser(bytes.NewBuffer(policyBytes))
return resp, nil
}
// Tests the processing of GetPolicy resposne from server.
// Tests the processing of GetPolicy response from server.
func TestProcessBucketPolicyResopnse(t *testing.T) {
bucketAccesPolicies := []BucketAccessPolicy{
{Version: "1.0"},
@ -139,6 +98,5 @@ func TestProcessBucketPolicyResopnse(t *testing.T) {
t.Errorf("Test %d: The expected BucketPolicy doesnt match the actual BucketPolicy", i+1)
}
}
}
}

4
vendor/src/github.com/minio/minio-go/api-list.go vendored
View file

@ -77,7 +77,7 @@ func (c Client) ListBuckets() ([]BucketInfo, error) {
//
func (c Client) ListObjects(bucketName, objectPrefix string, recursive bool, doneCh <-chan struct{}) <-chan ObjectInfo {
// Allocate new list objects channel.
objectStatCh := make(chan ObjectInfo, 1000)
objectStatCh := make(chan ObjectInfo)
// Default listing is delimited at "/"
delimiter := "/"
if recursive {
@ -254,7 +254,7 @@ func (c Client) ListIncompleteUploads(bucketName, objectPrefix string, recursive
// listIncompleteUploads lists all incomplete uploads.
func (c Client) listIncompleteUploads(bucketName, objectPrefix string, recursive, aggregateSize bool, doneCh <-chan struct{}) <-chan ObjectMultipartInfo {
// Allocate channel for multipart uploads.
objectMultipartStatCh := make(chan ObjectMultipartInfo, 1000)
objectMultipartStatCh := make(chan ObjectMultipartInfo)
// Delimiter is set to "/" by default.
delimiter := "/"
if recursive {

8
vendor/src/github.com/minio/minio-go/api-put-bucket_test.go vendored
View file

@ -27,9 +27,11 @@ import (
"testing"
)
// Tests validate http request formulated for creation of bucket.
func TestMakeBucketRequest(t *testing.T) {
// Generates expected http request for bucket creation.
// Used for asserting with the actual request generated.
func createExpectedRequest(c *Client, bucketName string, location string, req *http.Request) (*http.Request, error) {
createExpectedRequest := func(c *Client, bucketName string, location string, req *http.Request) (*http.Request, error) {
targetURL := *c.endpointURL
targetURL.Path = "/" + bucketName + "/"
@ -82,7 +84,7 @@ func createExpectedRequest(c *Client, bucketName string, location string, req *h
}
// Get Request body.
func getReqBody(reqBody io.ReadCloser) (string, error) {
getReqBody := func(reqBody io.ReadCloser) (string, error) {
contents, err := ioutil.ReadAll(reqBody)
if err != nil {
return "", err
@ -90,8 +92,6 @@ func getReqBody(reqBody io.ReadCloser) (string, error) {
return string(contents), nil
}
// Tests validate http request formulated for creation of bucket.
func TestMakeBucketRequest(t *testing.T) {
// Info for 'Client' creation.
// Will be used as arguments for 'NewClient'.
type infoForClient struct {

24
vendor/src/github.com/minio/minio-go/api.go vendored
View file

@ -419,11 +419,20 @@ func (c Client) executeMethod(method string, metadata requestMetadata) (res *htt
bodySeeker, isRetryable = metadata.contentBody.(io.Seeker)
}
// Create a done channel to control 'ListObjects' go routine.
doneCh := make(chan struct{}, 1)
// Indicate to our routine to exit cleanly upon return.
defer close(doneCh)
// Blank indentifier is kept here on purpose since 'range' without
// blank identifiers is only supported since go1.4
// https://golang.org/doc/go1.4#forrange.
for _ = range c.newRetryTimer(MaxRetry, time.Second, time.Second*30, MaxJitter, doneCh) {
// Retry executes the following function body if request has an
// error until maxRetries have been exhausted, retry attempts are
// performed after waiting for a given period of time in a
// binomial fashion.
for range c.newRetryTimer(MaxRetry, time.Second, time.Second*30, MaxJitter) {
if isRetryable {
// Seek back to beginning for each attempt.
if _, err = bodySeeker.Seek(0, 0); err != nil {
@ -505,8 +514,18 @@ func (c Client) newRequest(method string, metadata requestMetadata) (req *http.R
if method == "" {
method = "POST"
}
// Default all requests to "us-east-1" or "cn-north-1" (china region)
location := "us-east-1"
if isAmazonChinaEndpoint(c.endpointURL) {
// For china specifically we need to set everything to
// cn-north-1 for now, there is no easier way until AWS S3
// provides a cleaner compatible API across "us-east-1" and
// China region.
location = "cn-north-1"
}
// Gather location only if bucketName is present.
location := "us-east-1" // Default all other requests to "us-east-1".
if metadata.bucketName != "" {
location, err = c.getBucketLocation(metadata.bucketName)
if err != nil {
@ -648,6 +667,5 @@ func (c Client) makeTargetURL(bucketName, objectName, bucketLocation string, que
if err != nil {
return nil, err
}
return u, nil
}

19
vendor/src/github.com/minio/minio-go/bucket-cache.go vendored
View file

@ -72,6 +72,14 @@ func (c Client) getBucketLocation(bucketName string) (string, error) {
return location, nil
}
if isAmazonChinaEndpoint(c.endpointURL) {
// For china specifically we need to set everything to
// cn-north-1 for now, there is no easier way until AWS S3
// provides a cleaner compatible API across "us-east-1" and
// China region.
return "cn-north-1", nil
}
// Initialize a new request.
req, err := c.getBucketLocationRequest(bucketName)
if err != nil {
@ -84,6 +92,16 @@ func (c Client) getBucketLocation(bucketName string) (string, error) {
if err != nil {
return "", err
}
location, err := processBucketLocationResponse(resp, bucketName)
if err != nil {
return "", err
}
c.bucketLocCache.Set(bucketName, location)
return location, nil
}
// processes the getBucketLocation http response from the server.
func processBucketLocationResponse(resp *http.Response, bucketName string) (bucketLocation string, err error) {
if resp != nil {
if resp.StatusCode != http.StatusOK {
err = httpRespToErrorResponse(resp, bucketName, "")
@ -117,7 +135,6 @@ func (c Client) getBucketLocation(bucketName string) (string, error) {
}
// Save the location into cache.
c.bucketLocCache.Set(bucketName, location)
// Return.
return location, nil

320
vendor/src/github.com/minio/minio-go/bucket-cache_test.go vendored Normal file
View file

@ -0,0 +1,320 @@
/*
* Minio Go Library for Amazon S3 Compatible Cloud Storage (C) 2016, 2016 Minio, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package minio
import (
"bytes"
"encoding/hex"
"encoding/xml"
"io/ioutil"
"net/http"
"net/url"
"path"
"reflect"
"testing"
)
// Test validates `newBucketLocationCache`.
func TestNewBucketLocationCache(t *testing.T) {
expectedBucketLocationcache := &bucketLocationCache{
items: make(map[string]string),
}
actualBucketLocationCache := newBucketLocationCache()
if !reflect.DeepEqual(actualBucketLocationCache, expectedBucketLocationcache) {
t.Errorf("Unexpected return value")
}
}
// Tests validate bucketLocationCache operations.
func TestBucketLocationCacheOps(t *testing.T) {
testBucketLocationCache := newBucketLocationCache()
expectedBucketName := "minio-bucket"
expectedLocation := "us-east-1"
testBucketLocationCache.Set(expectedBucketName, expectedLocation)
actualLocation, ok := testBucketLocationCache.Get(expectedBucketName)
if !ok {
t.Errorf("Bucket location cache not set")
}
if expectedLocation != actualLocation {
t.Errorf("Bucket location cache not set to expected value")
}
testBucketLocationCache.Delete(expectedBucketName)
_, ok = testBucketLocationCache.Get(expectedBucketName)
if ok {
t.Errorf("Bucket location cache not deleted as expected")
}
}
// Tests validate http request generation for 'getBucketLocation'.
func TestGetBucketLocationRequest(t *testing.T) {
// Generates expected http request for getBucketLocation.
// Used for asserting with the actual request generated.
createExpectedRequest := func(c *Client, bucketName string, req *http.Request) (*http.Request, error) {
// Set location query.
urlValues := make(url.Values)
urlValues.Set("location", "")
// Set get bucket location always as path style.
targetURL := c.endpointURL
targetURL.Path = path.Join(bucketName, "") + "/"
targetURL.RawQuery = urlValues.Encode()
// Get a new HTTP request for the method.
req, err := http.NewRequest("GET", targetURL.String(), nil)
if err != nil {
return nil, err
}
// Set UserAgent for the request.
c.setUserAgent(req)
// Set sha256 sum for signature calculation only with signature version '4'.
if c.signature.isV4() {
req.Header.Set("X-Amz-Content-Sha256", hex.EncodeToString(sum256([]byte{})))
}
// Sign the request.
if c.signature.isV4() {
req = signV4(*req, c.accessKeyID, c.secretAccessKey, "us-east-1")
} else if c.signature.isV2() {
req = signV2(*req, c.accessKeyID, c.secretAccessKey)
}
return req, nil
}
// Info for 'Client' creation.
// Will be used as arguments for 'NewClient'.
type infoForClient struct {
endPoint string
accessKey string
secretKey string
enableInsecure bool
}
// dataset for 'NewClient' call.
info := []infoForClient{
// endpoint localhost.
// both access-key and secret-key are empty.
{"localhost:9000", "", "", false},
// both access-key are secret-key exists.
{"localhost:9000", "my-access-key", "my-secret-key", false},
// one of acess-key and secret-key are empty.
{"localhost:9000", "", "my-secret-key", false},
// endpoint amazon s3.
{"s3.amazonaws.com", "", "", false},
{"s3.amazonaws.com", "my-access-key", "my-secret-key", false},
{"s3.amazonaws.com", "my-acess-key", "", false},
// endpoint google cloud storage.
{"storage.googleapis.com", "", "", false},
{"storage.googleapis.com", "my-access-key", "my-secret-key", false},
{"storage.googleapis.com", "", "my-secret-key", false},
// endpoint custom domain running Minio server.
{"play.minio.io", "", "", false},
{"play.minio.io", "my-access-key", "my-secret-key", false},
{"play.minio.io", "my-acess-key", "", false},
}
testCases := []struct {
bucketName string
// data for new client creation.
info infoForClient
// error in the output.
err error
// flag indicating whether tests should pass.
shouldPass bool
}{
// Client is constructed using the info struct.
// case with empty location.
{"my-bucket", info[0], nil, true},
// case with location set to standard 'us-east-1'.
{"my-bucket", info[0], nil, true},
// case with location set to a value different from 'us-east-1'.
{"my-bucket", info[0], nil, true},
{"my-bucket", info[1], nil, true},
{"my-bucket", info[1], nil, true},
{"my-bucket", info[1], nil, true},
{"my-bucket", info[2], nil, true},
{"my-bucket", info[2], nil, true},
{"my-bucket", info[2], nil, true},
{"my-bucket", info[3], nil, true},
{"my-bucket", info[3], nil, true},
{"my-bucket", info[3], nil, true},
{"my-bucket", info[4], nil, true},
{"my-bucket", info[4], nil, true},
{"my-bucket", info[4], nil, true},
{"my-bucket", info[5], nil, true},
{"my-bucket", info[5], nil, true},
{"my-bucket", info[5], nil, true},
{"my-bucket", info[6], nil, true},
{"my-bucket", info[6], nil, true},
{"my-bucket", info[6], nil, true},
{"my-bucket", info[7], nil, true},
{"my-bucket", info[7], nil, true},
{"my-bucket", info[7], nil, true},
{"my-bucket", info[8], nil, true},
{"my-bucket", info[8], nil, true},
{"my-bucket", info[8], nil, true},
{"my-bucket", info[9], nil, true},
{"my-bucket", info[9], nil, true},
{"my-bucket", info[9], nil, true},
{"my-bucket", info[10], nil, true},
{"my-bucket", info[10], nil, true},
{"my-bucket", info[10], nil, true},
{"my-bucket", info[11], nil, true},
{"my-bucket", info[11], nil, true},
{"my-bucket", info[11], nil, true},
}
for i, testCase := range testCases {
// cannot create a newclient with empty endPoint value.
// validates and creates a new client only if the endPoint value is not empty.
client := &Client{}
var err error
if testCase.info.endPoint != "" {
client, err = New(testCase.info.endPoint, testCase.info.accessKey, testCase.info.secretKey, testCase.info.enableInsecure)
if err != nil {
t.Fatalf("Test %d: Failed to create new Client: %s", i+1, err.Error())
}
}
actualReq, err := client.getBucketLocationRequest(testCase.bucketName)
if err != nil && testCase.shouldPass {
t.Errorf("Test %d: Expected to pass, but failed with: <ERROR> %s", i+1, err.Error())
}
if err == nil && !testCase.shouldPass {
t.Errorf("Test %d: Expected to fail with <ERROR> \"%s\", but passed instead", i+1, testCase.err.Error())
}
// Failed as expected, but does it fail for the expected reason.
if err != nil && !testCase.shouldPass {
if err.Error() != testCase.err.Error() {
t.Errorf("Test %d: Expected to fail with error \"%s\", but instead failed with error \"%s\" instead", i+1, testCase.err.Error(), err.Error())
}
}
// Test passes as expected, but the output values are verified for correctness here.
if err == nil && testCase.shouldPass {
expectedReq := &http.Request{}
expectedReq, err = createExpectedRequest(client, testCase.bucketName, expectedReq)
if err != nil {
t.Fatalf("Test %d: Expected request Creation failed", i+1)
}
if expectedReq.Method != actualReq.Method {
t.Errorf("Test %d: The expected Request method doesn't match with the actual one", i+1)
}
if expectedReq.URL.String() != actualReq.URL.String() {
t.Errorf("Test %d: Expected the request URL to be '%s', but instead found '%s'", i+1, expectedReq.URL.String(), actualReq.URL.String())
}
if expectedReq.ContentLength != actualReq.ContentLength {
t.Errorf("Test %d: Expected the request body Content-Length to be '%d', but found '%d' instead", i+1, expectedReq.ContentLength, actualReq.ContentLength)
}
if expectedReq.Header.Get("X-Amz-Content-Sha256") != actualReq.Header.Get("X-Amz-Content-Sha256") {
t.Errorf("Test %d: 'X-Amz-Content-Sha256' header of the expected request doesn't match with that of the actual request", i+1)
}
if expectedReq.Header.Get("User-Agent") != actualReq.Header.Get("User-Agent") {
t.Errorf("Test %d: Expected 'User-Agent' header to be \"%s\",but found \"%s\" instead", i+1, expectedReq.Header.Get("User-Agent"), actualReq.Header.Get("User-Agent"))
}
}
}
}
// generates http response with bucket location set in the body.
func generateLocationResponse(resp *http.Response, bodyContent []byte) (*http.Response, error) {
resp.StatusCode = http.StatusOK
resp.Body = ioutil.NopCloser(bytes.NewBuffer(bodyContent))
return resp, nil
}
// Tests the processing of GetPolicy response from server.
func TestProcessBucketLocationResponse(t *testing.T) {
// LocationResponse - format for location response.
type LocationResponse struct {
XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ LocationConstraint" json:"-"`
Location string `xml:",chardata"`
}
APIErrors := []APIError{
{
Code: "AccessDenied",
Description: "Access Denied",
HTTPStatusCode: http.StatusUnauthorized,
},
}
testCases := []struct {
bucketName string
inputLocation string
isAPIError bool
apiErr APIError
// expected results.
expectedResult string
err error
// flag indicating whether tests should pass.
shouldPass bool
}{
{"my-bucket", "", true, APIErrors[0], "us-east-1", nil, true},
{"my-bucket", "", false, APIError{}, "us-east-1", nil, true},
{"my-bucket", "EU", false, APIError{}, "eu-west-1", nil, true},
{"my-bucket", "eu-central-1", false, APIError{}, "eu-central-1", nil, true},
{"my-bucket", "us-east-1", false, APIError{}, "us-east-1", nil, true},
}
for i, testCase := range testCases {
inputResponse := &http.Response{}
var err error
if testCase.isAPIError {
inputResponse = generateErrorResponse(inputResponse, testCase.apiErr, testCase.bucketName)
} else {
inputResponse, err = generateLocationResponse(inputResponse, encodeResponse(LocationResponse{
Location: testCase.inputLocation,
}))
if err != nil {
t.Fatalf("Test %d: Creation of valid response failed", i+1)
}
}
actualResult, err := processBucketLocationResponse(inputResponse, "my-bucket")
if err != nil && testCase.shouldPass {
t.Errorf("Test %d: Expected to pass, but failed with: <ERROR> %s", i+1, err.Error())
}
if err == nil && !testCase.shouldPass {
t.Errorf("Test %d: Expected to fail with <ERROR> \"%s\", but passed instead", i+1, testCase.err.Error())
}
// Failed as expected, but does it fail for the expected reason.
if err != nil && !testCase.shouldPass {
if err.Error() != testCase.err.Error() {
t.Errorf("Test %d: Expected to fail with error \"%s\", but instead failed with error \"%s\" instead", i+1, testCase.err.Error(), err.Error())
}
}
if err == nil && testCase.shouldPass {
if !reflect.DeepEqual(testCase.expectedResult, actualResult) {
t.Errorf("Test %d: The expected BucketPolicy doesnt match the actual BucketPolicy", i+1)
}
}
}
}

76
vendor/src/github.com/minio/minio-go/examples/s3/listobjects-N.go vendored Normal file
View file

@ -0,0 +1,76 @@
// +build ignore
/*
* Minio Go Library for Amazon S3 Compatible Cloud Storage (C) 2015 Minio, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package main
import (
"fmt"
"github.com/minio/minio-go"
)
func main() {
// Note: YOUR-ACCESSKEYID, YOUR-SECRETACCESSKEY, my-bucketname and my-prefixname
// are dummy values, please replace them with original values.
// Requests are always secure (HTTPS) by default. Set insecure=true to enable insecure (HTTP) access.
// This boolean value is the last argument for New().
// New returns an Amazon S3 compatible client object. API copatibality (v2 or v4) is automatically
// determined based on the Endpoint value.
s3Client, err := minio.New("s3.amazonaws.com", "YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", false)
if err != nil {
fmt.Println(err)
return
}
// List 'N' number of objects from a bucket-name with a matching prefix.
listObjectsN := func(bucket, prefix string, recursive bool, N int) (objsInfo []minio.ObjectInfo, err error) {
// Create a done channel to control 'ListObjects' go routine.
doneCh := make(chan struct{}, 1)
// Free the channel upon return.
defer close(doneCh)
i := 1
for object := range s3Client.ListObjects(bucket, prefix, recursive, doneCh) {
if object.Err != nil {
return nil, object.Err
}
i++
// Verify if we have printed N objects.
if i == N {
// Indicate ListObjects go-routine to exit and stop
// feeding the objectInfo channel.
doneCh <- struct{}{}
}
objsInfo = append(objsInfo, object)
}
return objsInfo, nil
}
// List recursively first 100 entries for prefix 'my-prefixname'.
recursive := true
objsInfo, err := listObjectsN("my-bucketname", "my-prefixname", recursive, 100)
if err != nil {
fmt.Println(err)
}
// Print all the entries.
fmt.Println(objsInfo)
}

21
vendor/src/github.com/minio/minio-go/retry.go vendored
View file

@ -35,7 +35,7 @@ const NoJitter = 0.0
// newRetryTimer creates a timer with exponentially increasing delays
// until the maximum retry attempts are reached.
func (c Client) newRetryTimer(maxRetry int, unit time.Duration, cap time.Duration, jitter float64) <-chan int {
func (c Client) newRetryTimer(maxRetry int, unit time.Duration, cap time.Duration, jitter float64, doneCh chan struct{}) <-chan int {
attemptCh := make(chan int)
// computes the exponential backoff duration according to
@ -63,7 +63,13 @@ func (c Client) newRetryTimer(maxRetry int, unit time.Duration, cap time.Duratio
go func() {
defer close(attemptCh)
for i := 0; i < maxRetry; i++ {
attemptCh <- i + 1 // Attempts start from 1.
select {
// Attempts start from 1.
case attemptCh <- i + 1:
case <-doneCh:
// Stop the routine.
return
}
time.Sleep(exponentialBackoffWait(i))
}
}()
@ -72,6 +78,8 @@ func (c Client) newRetryTimer(maxRetry int, unit time.Duration, cap time.Duratio
// isNetErrorRetryable - is network error retryable.
func isNetErrorRetryable(err error) bool {
switch err.(type) {
case net.Error:
switch err.(type) {
case *net.DNSError, *net.OpError, net.UnknownNetworkError:
return true
@ -81,6 +89,15 @@ func isNetErrorRetryable(err error) bool {
if strings.Contains(err.Error(), "Connection closed by foreign host") {
return true
}
default:
if strings.Contains(err.Error(), "net/http: TLS handshake timeout") {
// If error is - tlsHandshakeTimeoutError, retry.
return true
} else if strings.Contains(err.Error(), "i/o timeout") {
// If error is - tcp timeoutError, retry.
return true
}
}
}
return false
}

64
vendor/src/github.com/minio/minio-go/test-utils_test.go vendored Normal file
View file

@ -0,0 +1,64 @@
/*
* Minio Go Library for Amazon S3 Compatible Cloud Storage (C) 2015 Minio, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package minio
import (
"bytes"
"encoding/xml"
"io/ioutil"
"net/http"
)
// Contains common used utilities for tests.
// APIError Used for mocking error response from server.
type APIError struct {
Code string
Description string
HTTPStatusCode int
}
// Mocks XML error response from the server.
func generateErrorResponse(resp *http.Response, APIErr APIError, bucketName string) *http.Response {
// generate error response.
errorResponse := getAPIErrorResponse(APIErr, bucketName)
encodedErrorResponse := encodeResponse(errorResponse)
// write Header.
resp.StatusCode = APIErr.HTTPStatusCode
resp.Body = ioutil.NopCloser(bytes.NewBuffer(encodedErrorResponse))
return resp
}
// getErrorResponse gets in standard error and resource value and
// provides a encodable populated response values.
func getAPIErrorResponse(err APIError, bucketName string) ErrorResponse {
var errResp = ErrorResponse{}
errResp.Code = err.Code
errResp.Message = err.Description
errResp.BucketName = bucketName
return errResp
}
// Encodes the response headers into XML format.
func encodeResponse(response interface{}) []byte {
var bytesBuffer bytes.Buffer
bytesBuffer.WriteString(xml.Header)
encode := xml.NewEncoder(&bytesBuffer)
encode.Encode(response)
return bytesBuffer.Bytes()
}