Merge pull request #1010 from restic/update-minio-go

Update github.com/minio/minio-go

src/restic/backend/s3/s3_test.go

@@ -49,7 +49,7 @@ func runMinio(ctx context.Context, t testing.TB, dir, key, secret string) func()
 
 	// wait until the TCP port is reachable
 	var success bool
-	for i := 0; i < 10; i++ {
+	for i := 0; i < 100; i++ {
 		time.Sleep(200 * time.Millisecond)
 
 		c, err := net.Dial("tcp", "localhost:9000")

vendor/manifest

@@ -46,7 +46,7 @@
 		{
 			"importpath": "github.com/minio/minio-go",
 			"repository": "https://github.com/minio/minio-go",
-			"revision": "85f15b007f08e11a62c769abe65299b812fd2e0d",
+			"revision": "f2362d9e7d8daf89594ee0a079be2424eaf360be",
 			"branch": "master"
 		},
 		{

vendor/src/github.com/minio/minio-go/api-get-object.go

@@ -28,9 +28,9 @@ import (
 	"github.com/minio/minio-go/pkg/encrypt"
 )
 
-// GetEncryptedObject deciphers and streams data stored in the server after applying a specifed encryption materiels
-func (c Client) GetEncryptedObject(bucketName, objectName string, encryptMaterials encrypt.Materials) (io.Reader, error) {
-
+// GetEncryptedObject deciphers and streams data stored in the server after applying a specifed encryption materials,
+// returned stream should be closed by the caller.
+func (c Client) GetEncryptedObject(bucketName, objectName string, encryptMaterials encrypt.Materials) (io.ReadCloser, error) {
 	if encryptMaterials == nil {
 		return nil, ErrInvalidArgument("Unable to recognize empty encryption properties")
 	}
@@ -328,14 +328,14 @@ func (o *Object) setOffset(bytesRead int64) error {
 	// Update the currentOffset.
 	o.currOffset += bytesRead
 
-	if o.currOffset >= o.objectInfo.Size {
+	if o.objectInfo.Size > -1 && o.currOffset >= o.objectInfo.Size {
 		return io.EOF
 	}
 	return nil
 }
 
 // Read reads up to len(b) bytes into b. It returns the number of
-// bytes read (0 <= n <= len(p)) and any error encountered. Returns
+// bytes read (0 <= n <= len(b)) and any error encountered. Returns
 // io.EOF upon end of file.
 func (o *Object) Read(b []byte) (n int, err error) {
 	if o == nil {
@@ -442,7 +442,7 @@ func (o *Object) ReadAt(b []byte, offset int64) (n int, err error) {
 	if o.objectInfoSet {
 		// If offset is negative than we return io.EOF.
 		// If offset is greater than or equal to object size we return io.EOF.
-		if offset >= o.objectInfo.Size || offset < 0 {
+		if (o.objectInfo.Size > -1 && offset >= o.objectInfo.Size) || offset < 0 {
 			return 0, io.EOF
 		}
 	}
@@ -542,16 +542,20 @@ func (o *Object) Seek(offset int64, whence int) (n int64, err error) {
 	default:
 		return 0, ErrInvalidArgument(fmt.Sprintf("Invalid whence %d", whence))
 	case 0:
-		if offset > o.objectInfo.Size {
+		if o.objectInfo.Size > -1 && offset > o.objectInfo.Size {
 			return 0, io.EOF
 		}
 		o.currOffset = offset
 	case 1:
-		if o.currOffset+offset > o.objectInfo.Size {
+		if o.objectInfo.Size > -1 && o.currOffset+offset > o.objectInfo.Size {
 			return 0, io.EOF
 		}
 		o.currOffset += offset
 	case 2:
+		// If we don't know the object size return an error for io.SeekEnd
+		if o.objectInfo.Size < 0 {
+			return 0, ErrInvalidArgument("Whence END is not supported when the object size is unknown")
+		}
 		// Seeking to positive offset is valid for whence '2', but
 		// since we are backing a Reader we have reached 'EOF' if
 		// offset is positive.

vendor/src/github.com/minio/minio-go/api-stat.go

@@ -21,8 +21,6 @@ import (
 	"strconv"
 	"strings"
 	"time"
-
-	"github.com/minio/minio-go/pkg/s3utils"
 )
 
 // BucketExists verify if bucket exists and you have permission to access it.
@@ -126,12 +124,13 @@ func (c Client) statObject(bucketName, objectName string, reqHeaders RequestHead
 	md5sum := strings.TrimPrefix(resp.Header.Get("ETag"), "\"")
 	md5sum = strings.TrimSuffix(md5sum, "\"")
 
-	// Content-Length is not valid for Google Cloud Storage, do not verify.
+	// Parse content length is exists
 	var size int64 = -1
-	if !s3utils.IsGoogleEndpoint(c.endpointURL) {
-		// Parse content length.
-		size, err = strconv.ParseInt(resp.Header.Get("Content-Length"), 10, 64)
+	contentLengthStr := resp.Header.Get("Content-Length")
+	if contentLengthStr != "" {
+		size, err = strconv.ParseInt(contentLengthStr, 10, 64)
 		if err != nil {
+			// Content-Length is not valid
 			return ObjectInfo{}, ErrorResponse{
 				Code:       "InternalError",
 				Message:    "Content-Length is invalid. " + reportIssue,

vendor/src/github.com/minio/minio-go/api_functional_v4_test.go

@@ -1962,6 +1962,7 @@ func TestEncryptionPutGet(t *testing.T) {
 		if err != nil {
 			t.Fatalf("Test %d, error: %v %v %v", i+1, err, bucketName, objectName)
 		}
+		defer r.Close()
 
 		// Compare the sent object with the received one
 		recvBuffer := bytes.NewBuffer([]byte{})

vendor/src/github.com/minio/minio-go/docs/API.md

@@ -771,7 +771,7 @@ if err != nil {
 ```
 
 <a name="GetEncryptedObject"></a>
-### GetEncryptedObject(bucketName, objectName string, encryptMaterials minio.EncryptionMaterials) (io.Reader, error)
+### GetEncryptedObject(bucketName, objectName string, encryptMaterials minio.EncryptionMaterials) (io.ReadCloser, error)
 
 Returns the decrypted stream of the object data based of the given encryption materiels. Most of the common errors occur when reading the stream.
 
@@ -788,7 +788,7 @@ __Return Value__
 
 |Param   |Type   |Description   |
 |:---|:---| :---|
-|`stream`  | _io.Reader_ | Returns the deciphered object reader. |
+|`stream`  | _io.ReadCloser_ | Returns the deciphered object reader, caller should close after reading. |
 |`err`  | _error | Returns errors. |
 
 
@@ -810,11 +810,14 @@ if err != nil {
     fmt.Println(err)
     return
 }
+defer object.Close()
+
 localFile, err := os.Create("/tmp/local-file.jpg")
 if err != nil {
     fmt.Println(err)
     return
 }
+
 if _, err = io.Copy(localFile, object); err != nil {
     fmt.Println(err)
     return

vendor/src/github.com/minio/minio-go/examples/s3/get-encrypted-object.go

@@ -24,6 +24,7 @@ import (
 	"os"
 
 	"github.com/minio/minio-go"
+	"github.com/minio/minio-go/pkg/encrypt"
 )
 
 func main() {
@@ -59,10 +60,10 @@ func main() {
 	////
 
 	// Build a symmetric key
-	symmetricKey := minio.NewSymmetricKey([]byte("my-secret-key-00"))
+	symmetricKey := encrypt.NewSymmetricKey([]byte("my-secret-key-00"))
 
 	// Build encryption materials which will encrypt uploaded data
-	cbcMaterials, err := minio.NewCBCSecureMaterials(symmetricKey)
+	cbcMaterials, err := encrypt.NewCBCSecureMaterials(symmetricKey)
 	if err != nil {
 		log.Fatalln(err)
 	}
@@ -72,6 +73,7 @@ func main() {
 	if err != nil {
 		log.Fatalln(err)
 	}
+	defer reader.Close()
 
 	// Local file which holds plain data
 	localFile, err := os.Create("my-testfile")

vendor/src/github.com/minio/minio-go/examples/s3/put-encrypted-object.go

@@ -23,6 +23,7 @@ import (
 	"os"
 
 	"github.com/minio/minio-go"
+	"github.com/minio/minio-go/pkg/encrypt"
 )
 
 func main() {
@@ -65,10 +66,10 @@ func main() {
 	////
 
 	// Build a symmetric key
-	symmetricKey := minio.NewSymmetricKey([]byte("my-secret-key-00"))
+	symmetricKey := encrypt.NewSymmetricKey([]byte("my-secret-key-00"))
 
 	// Build encryption materials which will encrypt uploaded data
-	cbcMaterials, err := minio.NewCBCSecureMaterials(symmetricKey)
+	cbcMaterials, err := encrypt.NewCBCSecureMaterials(symmetricKey)
 	if err != nil {
 		log.Fatalln(err)
 	}

vendor/src/github.com/minio/minio-go/examples/s3/putobject-getobject-sse.go

@@ -0,0 +1,87 @@
+// +build ignore
+
+/*
+ * Minio Go Library for Amazon S3 Compatible Cloud Storage (C) 2017 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package main
+
+import (
+	"bytes"
+	"crypto/md5"
+	"encoding/base64"
+	"io/ioutil"
+	"log"
+	"net/http"
+
+	minio "github.com/minio/minio-go"
+)
+
+func main() {
+	// Note: YOUR-ACCESSKEYID, YOUR-SECRETACCESSKEY, my-testfile, my-bucketname and
+	// my-objectname are dummy values, please replace them with original values.
+
+	// New returns an Amazon S3 compatible client object. API compatibility (v2 or v4) is automatically
+	// determined based on the Endpoint value.
+	minioClient, err := minio.New("s3.amazonaws.com", "YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", true)
+	if err != nil {
+		log.Fatalln(err)
+	}
+
+	content := bytes.NewReader([]byte("Hello again"))
+	key := []byte("32byteslongsecretkeymustprovided")
+	h := md5.New()
+	h.Write(key)
+	encryptionKey := base64.StdEncoding.EncodeToString(key)
+	encryptionKeyMD5 := base64.StdEncoding.EncodeToString(h.Sum(nil))
+
+	// Amazon S3 does not store the encryption key you provide.
+	// Instead S3 stores a randomly salted HMAC value of the
+	// encryption key in order to validate future requests.
+	// The salted HMAC value cannot be used to derive the value
+	// of the encryption key or to decrypt the contents of the
+	// encrypted object. That means, if you lose the encryption
+	// key, you lose the object.
+	var metadata = map[string][]string{
+		"x-amz-server-side-encryption-customer-algorithm": []string{"AES256"},
+		"x-amz-server-side-encryption-customer-key":       []string{encryptionKey},
+		"x-amz-server-side-encryption-customer-key-MD5":   []string{encryptionKeyMD5},
+	}
+
+	// minioClient.TraceOn(os.Stderr) // Enable to debug.
+	_, err = minioClient.PutObjectWithMetadata("mybucket", "my-encrypted-object.txt", content, metadata, nil)
+	if err != nil {
+		log.Fatalln(err)
+	}
+
+	var reqHeaders = minio.RequestHeaders{Header: http.Header{}}
+	for k, v := range metadata {
+		reqHeaders.Set(k, v[0])
+	}
+	coreClient := minio.Core{minioClient}
+	reader, _, err := coreClient.GetObject("mybucket", "my-encrypted-object.txt", reqHeaders)
+	if err != nil {
+		log.Fatalln(err)
+	}
+	defer reader.Close()
+
+	decBytes, err := ioutil.ReadAll(reader)
+	if err != nil {
+		log.Fatalln(err)
+	}
+	if !bytes.Equal(decBytes, []byte("Hello again")) {
+		log.Fatalln("Expected \"Hello, world\", got %s", string(decBytes))
+	}
+}

vendor/src/github.com/minio/minio-go/pkg/encrypt/cbc.go

@@ -89,6 +89,15 @@ func NewCBCSecureMaterials(key Key) (*CBCSecureMaterials, error) {
 
 }
 
+// Close implements closes the internal stream.
+func (s *CBCSecureMaterials) Close() error {
+	closer, ok := s.stream.(io.Closer)
+	if ok {
+		return closer.Close()
+	}
+	return nil
+}
+
 // SetupEncryptMode - tells CBC that we are going to encrypt data
 func (s *CBCSecureMaterials) SetupEncryptMode(stream io.Reader) error {
 	// Set mode to encrypt

vendor/src/github.com/minio/minio-go/pkg/encrypt/interface.go

@@ -25,6 +25,9 @@ import "io"
 // Materials - provides generic interface to encrypt any stream of data.
 type Materials interface {
 
+	// Closes the wrapped stream properly, initiated by the caller.
+	Close() error
+
 	// Returns encrypted/decrypted data, io.Reader compatible.
 	Read(b []byte) (int, error)
 

vendor/src/github.com/minio/minio-go/request-headers.go

@@ -48,7 +48,7 @@ func (c RequestHeaders) SetMatchETag(etag string) error {
 	if etag == "" {
 		return ErrInvalidArgument("ETag cannot be empty.")
 	}
-	c.Set("If-Match", etag)
+	c.Set("If-Match", "\""+etag+"\"")
 	return nil
 }
 
@@ -57,7 +57,7 @@ func (c RequestHeaders) SetMatchETagExcept(etag string) error {
 	if etag == "" {
 		return ErrInvalidArgument("ETag cannot be empty.")
 	}
-	c.Set("If-None-Match", etag)
+	c.Set("If-None-Match", "\""+etag+"\"")
 	return nil
 }
 

vendor/src/github.com/minio/minio-go/s3-error.go

@@ -25,7 +25,7 @@ var s3ErrorResponseMap = map[string]string{
 	"EntityTooLarge":                    "Your proposed upload exceeds the maximum allowed object size.",
 	"IncompleteBody":                    "You did not provide the number of bytes specified by the Content-Length HTTP header.",
 	"InternalError":                     "We encountered an internal error, please try again.",
-	"InvalidAccessKeyID":                "The access key ID you provided does not exist in our records.",
+	"InvalidAccessKeyId":                "The access key ID you provided does not exist in our records.",
 	"InvalidBucketName":                 "The specified bucket is not valid.",
 	"InvalidDigest":                     "The Content-Md5 you specified is not valid.",
 	"InvalidRange":                      "The requested range is not satisfiable",