Switch to using the dep tool and update all the dependencies

vendor/github.com/aws/aws-sdk-go/awstesting/integration/customizations/s3/integration_test.go

@@ -0,0 +1,124 @@
+// +build integration
+
+// Package s3_test runs integration tests for S3
+package s3_test
+
+import (
+	"bytes"
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"os"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/awstesting/integration"
+	"github.com/aws/aws-sdk-go/service/s3"
+)
+
+var bucketName *string
+var svc *s3.S3
+
+func TestMain(m *testing.M) {
+	setup()
+	defer teardown() // only called if we panic
+	result := m.Run()
+	teardown()
+	os.Exit(result)
+}
+
+// Create a bucket for testing
+func setup() {
+	svc = s3.New(integration.Session)
+	bucketName = aws.String(
+		fmt.Sprintf("aws-sdk-go-integration-%d-%s", time.Now().Unix(), integration.UniqueID()))
+
+	for i := 0; i < 10; i++ {
+		_, err := svc.CreateBucket(&s3.CreateBucketInput{Bucket: bucketName})
+		if err == nil {
+			break
+		}
+	}
+
+	for {
+		_, err := svc.HeadBucket(&s3.HeadBucketInput{Bucket: bucketName})
+		if err == nil {
+			break
+		}
+		time.Sleep(1 * time.Second)
+	}
+}
+
+// Delete the bucket
+func teardown() {
+	resp, _ := svc.ListObjects(&s3.ListObjectsInput{Bucket: bucketName})
+	for _, o := range resp.Contents {
+		svc.DeleteObject(&s3.DeleteObjectInput{Bucket: bucketName, Key: o.Key})
+	}
+	svc.DeleteBucket(&s3.DeleteBucketInput{Bucket: bucketName})
+}
+
+func TestWriteToObject(t *testing.T) {
+	_, err := svc.PutObject(&s3.PutObjectInput{
+		Bucket: bucketName,
+		Key:    aws.String("key name"),
+		Body:   bytes.NewReader([]byte("hello world")),
+	})
+	assert.NoError(t, err)
+
+	resp, err := svc.GetObject(&s3.GetObjectInput{
+		Bucket: bucketName,
+		Key:    aws.String("key name"),
+	})
+	assert.NoError(t, err)
+
+	b, _ := ioutil.ReadAll(resp.Body)
+	assert.Equal(t, []byte("hello world"), b)
+}
+
+func TestPresignedGetPut(t *testing.T) {
+	putreq, _ := svc.PutObjectRequest(&s3.PutObjectInput{
+		Bucket: bucketName,
+		Key:    aws.String("presigned-key"),
+	})
+	var err error
+
+	// Presign a PUT request
+	var puturl string
+	puturl, err = putreq.Presign(300 * time.Second)
+	assert.NoError(t, err)
+
+	// PUT to the presigned URL with a body
+	var puthttpreq *http.Request
+	buf := bytes.NewReader([]byte("hello world"))
+	puthttpreq, err = http.NewRequest("PUT", puturl, buf)
+	assert.NoError(t, err)
+
+	var putresp *http.Response
+	putresp, err = http.DefaultClient.Do(puthttpreq)
+	assert.NoError(t, err)
+	assert.Equal(t, 200, putresp.StatusCode)
+
+	// Presign a GET on the same URL
+	getreq, _ := svc.GetObjectRequest(&s3.GetObjectInput{
+		Bucket: bucketName,
+		Key:    aws.String("presigned-key"),
+	})
+
+	var geturl string
+	geturl, err = getreq.Presign(300 * time.Second)
+	assert.NoError(t, err)
+
+	// Get the body
+	var getresp *http.Response
+	getresp, err = http.Get(geturl)
+	assert.NoError(t, err)
+
+	var b []byte
+	defer getresp.Body.Close()
+	b, err = ioutil.ReadAll(getresp.Body)
+	assert.Equal(t, "hello world", string(b))
+}

vendor/github.com/aws/aws-sdk-go/awstesting/integration/customizations/s3/s3crypto/client.go

@@ -0,0 +1,29 @@
+// +build integration
+
+//Package s3crypto provides gucumber integration tests support.
+package s3crypto
+
+import (
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/session"
+	"github.com/aws/aws-sdk-go/service/s3"
+	"github.com/aws/aws-sdk-go/service/s3/s3crypto"
+
+	"github.com/gucumber/gucumber"
+)
+
+func init() {
+	gucumber.Before("@s3crypto", func() {
+		sess := session.New((&aws.Config{
+			Region: aws.String("us-west-2"),
+		}))
+		encryptionClient := s3crypto.NewEncryptionClient(sess, nil, func(c *s3crypto.EncryptionClient) {
+		})
+		gucumber.World["encryptionClient"] = encryptionClient
+
+		decryptionClient := s3crypto.NewDecryptionClient(sess)
+		gucumber.World["decryptionClient"] = decryptionClient
+
+		gucumber.World["client"] = s3.New(sess)
+	})
+}

vendor/github.com/aws/aws-sdk-go/awstesting/integration/customizations/s3/s3crypto/s3_crypto.feature

@@ -0,0 +1,33 @@
+# language: en
+@s3crypto @client
+Feature: S3 Integration Crypto Tests
+
+  Scenario: Uploading Go's SDK fixtures
+    When I get all fixtures for "aes_gcm" from "aws-s3-shared-tests"
+    Then I encrypt each fixture with "kms" "AWS_SDK_TEST_ALIAS" "us-west-2" and "aes_gcm"
+    And upload "Go" data with folder "version_2"
+
+  Scenario: Uploading Go's SDK fixtures
+    When I get all fixtures for "aes_cbc" from "aws-s3-shared-tests"
+    Then I encrypt each fixture with "kms" "AWS_SDK_TEST_ALIAS" "us-west-2" and "aes_cbc"
+    And upload "Go" data with folder "version_2"
+
+  Scenario: Get all plaintext fixtures for symmetric masterkey aes gcm 
+    When I get all fixtures for "aes_gcm" from "aws-s3-shared-tests"
+    Then I decrypt each fixture against "Go" "version_2"
+    And I compare the decrypted ciphertext to the plaintext
+
+  Scenario: Get all plaintext fixtures for symmetric masterkey aes cbc
+    When I get all fixtures for "aes_cbc" from "aws-s3-shared-tests"
+    Then I decrypt each fixture against "Go" "version_2"
+    And I compare the decrypted ciphertext to the plaintext
+
+  Scenario: Get all plaintext fixtures for symmetric masterkey aes gcm
+    When I get all fixtures for "aes_gcm" from "aws-s3-shared-tests"
+    Then I decrypt each fixture against "Java" "version_2"
+    And I compare the decrypted ciphertext to the plaintext
+
+  Scenario: Get all plaintext fixtures for symmetric masterkey aes cbc
+    When I get all fixtures for "aes_cbc" from "aws-s3-shared-tests"
+    Then I decrypt each fixture against "Java" "version_2"
+    And I compare the decrypted ciphertext to the plaintext

vendor/github.com/aws/aws-sdk-go/awstesting/integration/customizations/s3/s3crypto/stepdef.go

@@ -0,0 +1,190 @@
+// +build integration
+
+// Package s3crypto contains shared step definitions that are used across integration tests
+package s3crypto
+
+import (
+	"bytes"
+	"encoding/base64"
+	"errors"
+	"io/ioutil"
+	"strings"
+
+	"github.com/gucumber/gucumber"
+	"github.com/stretchr/testify/assert"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/session"
+	"github.com/aws/aws-sdk-go/service/kms"
+	"github.com/aws/aws-sdk-go/service/s3"
+	"github.com/aws/aws-sdk-go/service/s3/s3crypto"
+)
+
+func init() {
+	gucumber.When(`^I get all fixtures for "(.+?)" from "(.+?)"$`,
+		func(cekAlg, bucket string) {
+			prefix := "plaintext_test_case_"
+			baseFolder := "crypto_tests/" + cekAlg
+			s3Client := gucumber.World["client"].(*s3.S3)
+
+			out, err := s3Client.ListObjects(&s3.ListObjectsInput{
+				Bucket: aws.String(bucket),
+				Prefix: aws.String(baseFolder + "/" + prefix),
+			})
+			assert.NoError(gucumber.T, err)
+
+			plaintexts := make(map[string][]byte)
+			for _, obj := range out.Contents {
+				plaintextKey := obj.Key
+				ptObj, err := s3Client.GetObject(&s3.GetObjectInput{
+					Bucket: aws.String(bucket),
+					Key:    plaintextKey,
+				})
+				assert.NoError(gucumber.T, err)
+				caseKey := strings.TrimPrefix(*plaintextKey, baseFolder+"/"+prefix)
+				plaintext, err := ioutil.ReadAll(ptObj.Body)
+				assert.NoError(gucumber.T, err)
+
+				plaintexts[caseKey] = plaintext
+			}
+			gucumber.World["baseFolder"] = baseFolder
+			gucumber.World["bucket"] = bucket
+			gucumber.World["plaintexts"] = plaintexts
+		})
+
+	gucumber.Then(`^I decrypt each fixture against "(.+?)" "(.+?)"$`, func(lang, version string) {
+		plaintexts := gucumber.World["plaintexts"].(map[string][]byte)
+		baseFolder := gucumber.World["baseFolder"].(string)
+		bucket := gucumber.World["bucket"].(string)
+		prefix := "ciphertext_test_case_"
+		s3Client := gucumber.World["client"].(*s3.S3)
+		s3CryptoClient := gucumber.World["decryptionClient"].(*s3crypto.DecryptionClient)
+		language := "language_" + lang
+
+		ciphertexts := make(map[string][]byte)
+		for caseKey := range plaintexts {
+			cipherKey := baseFolder + "/" + version + "/" + language + "/" + prefix + caseKey
+
+			// To get metadata for encryption key
+			ctObj, err := s3Client.GetObject(&s3.GetObjectInput{
+				Bucket: aws.String(bucket),
+				Key:    &cipherKey,
+			})
+			if err != nil {
+				continue
+			}
+
+			// We don't support wrap, so skip it
+			if ctObj.Metadata["X-Amz-Wrap-Alg"] == nil || *ctObj.Metadata["X-Amz-Wrap-Alg"] != "kms" {
+				continue
+			}
+
+			ctObj, err = s3CryptoClient.GetObject(&s3.GetObjectInput{
+				Bucket: aws.String(bucket),
+				Key:    &cipherKey,
+			},
+			)
+			assert.NoError(gucumber.T, err)
+
+			ciphertext, err := ioutil.ReadAll(ctObj.Body)
+			assert.NoError(gucumber.T, err)
+			ciphertexts[caseKey] = ciphertext
+		}
+		gucumber.World["decrypted"] = ciphertexts
+	})
+
+	gucumber.And(`^I compare the decrypted ciphertext to the plaintext$`, func() {
+		plaintexts := gucumber.World["plaintexts"].(map[string][]byte)
+		ciphertexts := gucumber.World["decrypted"].(map[string][]byte)
+		for caseKey, ciphertext := range ciphertexts {
+			assert.Equal(gucumber.T, len(plaintexts[caseKey]), len(ciphertext))
+			assert.True(gucumber.T, bytes.Equal(plaintexts[caseKey], ciphertext))
+		}
+	})
+
+	gucumber.Then(`^I encrypt each fixture with "(.+?)" "(.+?)" "(.+?)" and "(.+?)"$`, func(kek, v1, v2, cek string) {
+		var handler s3crypto.CipherDataGenerator
+		var builder s3crypto.ContentCipherBuilder
+		switch kek {
+		case "kms":
+			arn, err := getAliasInformation(v1, v2)
+			assert.Nil(gucumber.T, err)
+
+			b64Arn := base64.StdEncoding.EncodeToString([]byte(arn))
+			assert.Nil(gucumber.T, err)
+			gucumber.World["Masterkey"] = b64Arn
+
+			handler = s3crypto.NewKMSKeyGenerator(kms.New(session.New(&aws.Config{
+				Region: &v2,
+			})), arn)
+			assert.Nil(gucumber.T, err)
+		default:
+			gucumber.T.Skip()
+		}
+
+		switch cek {
+		case "aes_gcm":
+			builder = s3crypto.AESGCMContentCipherBuilder(handler)
+		case "aes_cbc":
+			builder = s3crypto.AESCBCContentCipherBuilder(handler, s3crypto.AESCBCPadder)
+		default:
+			gucumber.T.Skip()
+		}
+
+		sess := session.New(&aws.Config{
+			Region: aws.String("us-west-2"),
+		})
+		c := s3crypto.NewEncryptionClient(sess, builder, func(c *s3crypto.EncryptionClient) {
+		})
+		gucumber.World["encryptionClient"] = c
+		gucumber.World["cek"] = cek
+	})
+
+	gucumber.And(`^upload "(.+?)" data with folder "(.+?)"$`, func(language, folder string) {
+		c := gucumber.World["encryptionClient"].(*s3crypto.EncryptionClient)
+		cek := gucumber.World["cek"].(string)
+		bucket := gucumber.World["bucket"].(string)
+		plaintexts := gucumber.World["plaintexts"].(map[string][]byte)
+		key := gucumber.World["Masterkey"].(string)
+		for caseKey, plaintext := range plaintexts {
+			input := &s3.PutObjectInput{
+				Bucket: &bucket,
+				Key:    aws.String("crypto_tests/" + cek + "/" + folder + "/language_" + language + "/ciphertext_test_case_" + caseKey),
+				Body:   bytes.NewReader(plaintext),
+				Metadata: map[string]*string{
+					"Masterkey": &key,
+				},
+			}
+
+			_, err := c.PutObject(input)
+			assert.Nil(gucumber.T, err)
+		}
+	})
+}
+
+func getAliasInformation(alias, region string) (string, error) {
+	arn := ""
+	svc := kms.New(session.New(&aws.Config{
+		Region: &region,
+	}))
+
+	truncated := true
+	var marker *string
+	for truncated {
+		out, err := svc.ListAliases(&kms.ListAliasesInput{
+			Marker: marker,
+		})
+		if err != nil {
+			return arn, err
+		}
+		for _, aliasEntry := range out.Aliases {
+			if *aliasEntry.AliasName == "alias/"+alias {
+				return *aliasEntry.AliasArn, nil
+			}
+		}
+		truncated = *out.Truncated
+		marker = out.NextMarker
+	}
+
+	return "", errors.New("The alias " + alias + " does not exist in your account. Please add the proper alias to a key")
+}

vendor/github.com/aws/aws-sdk-go/awstesting/integration/customizations/s3/s3manager/bucket_region_test.go

@@ -0,0 +1,27 @@
+// +build integration
+
+package s3manager
+
+import (
+	"testing"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/awstesting/integration"
+	"github.com/aws/aws-sdk-go/service/s3/s3manager"
+)
+
+func TestGetBucketRegion(t *testing.T) {
+	expectRegion := aws.StringValue(integration.Session.Config.Region)
+
+	ctx := aws.BackgroundContext()
+	region, err := s3manager.GetBucketRegion(ctx, integration.Session,
+		aws.StringValue(bucketName), expectRegion)
+
+	if err != nil {
+		t.Fatalf("expect no error, got %v", err)
+	}
+
+	if e, a := expectRegion, region; e != a {
+		t.Errorf("expect %s bucket region, got %s", e, a)
+	}
+}

vendor/github.com/aws/aws-sdk-go/awstesting/integration/customizations/s3/s3manager/integration_test.go

@@ -0,0 +1,210 @@
+// +build integration
+
+// Package s3manager provides integration tests for the service/s3/s3manager package
+package s3manager
+
+import (
+	"bytes"
+	"crypto/md5"
+	"fmt"
+	"io"
+	"os"
+	"regexp"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/awserr"
+	"github.com/aws/aws-sdk-go/aws/request"
+	"github.com/aws/aws-sdk-go/awstesting/integration"
+	"github.com/aws/aws-sdk-go/service/s3"
+	"github.com/aws/aws-sdk-go/service/s3/s3manager"
+)
+
+var integBuf12MB = make([]byte, 1024*1024*12)
+var integMD512MB = fmt.Sprintf("%x", md5.Sum(integBuf12MB))
+var bucketName *string
+
+func TestMain(m *testing.M) {
+	if err := setup(); err != nil {
+		panic(fmt.Sprintf("failed to setup integration test, %v", err))
+	}
+
+	var result int
+
+	defer func() {
+		if err := teardown(); err != nil {
+			fmt.Fprintf(os.Stderr, "teardown failed, %v", err)
+		}
+		if r := recover(); r != nil {
+			fmt.Println("S3Manager integration test hit a panic,", r)
+			result = 1
+		}
+		os.Exit(result)
+	}()
+
+	result = m.Run()
+}
+
+func setup() error {
+	svc := s3.New(integration.Session)
+
+	// Create a bucket for testing
+	bucketName = aws.String(
+		fmt.Sprintf("aws-sdk-go-integration-%d-%s", time.Now().Unix(), integration.UniqueID()))
+
+	_, err := svc.CreateBucket(&s3.CreateBucketInput{Bucket: bucketName})
+	if err != nil {
+		return fmt.Errorf("failed to create bucket %q, %v", *bucketName, err)
+	}
+
+	err = svc.WaitUntilBucketExists(&s3.HeadBucketInput{Bucket: bucketName})
+	if err != nil {
+		return fmt.Errorf("failed to wait for bucket %q to exist, %v", bucketName, err)
+	}
+
+	return nil
+}
+
+// Delete the bucket
+func teardown() error {
+	svc := s3.New(integration.Session)
+
+	objs, err := svc.ListObjects(&s3.ListObjectsInput{Bucket: bucketName})
+	if err != nil {
+		return fmt.Errorf("failed to list bucket %q objects, %v", bucketName, err)
+	}
+
+	for _, o := range objs.Contents {
+		svc.DeleteObject(&s3.DeleteObjectInput{Bucket: bucketName, Key: o.Key})
+	}
+
+	uploads, err := svc.ListMultipartUploads(&s3.ListMultipartUploadsInput{Bucket: bucketName})
+	if err != nil {
+		return fmt.Errorf("failed to list bucket %q multipart objects, %v", bucketName, err)
+	}
+
+	for _, u := range uploads.Uploads {
+		svc.AbortMultipartUpload(&s3.AbortMultipartUploadInput{
+			Bucket:   bucketName,
+			Key:      u.Key,
+			UploadId: u.UploadId,
+		})
+	}
+
+	_, err = svc.DeleteBucket(&s3.DeleteBucketInput{Bucket: bucketName})
+	if err != nil {
+		return fmt.Errorf("failed to delete bucket %q, %v", bucketName, err)
+	}
+
+	return nil
+}
+
+type dlwriter struct {
+	buf []byte
+}
+
+func newDLWriter(size int) *dlwriter {
+	return &dlwriter{buf: make([]byte, size)}
+}
+
+func (d dlwriter) WriteAt(p []byte, pos int64) (n int, err error) {
+	if pos > int64(len(d.buf)) {
+		return 0, io.EOF
+	}
+
+	written := 0
+	for i, b := range p {
+		if i >= len(d.buf) {
+			break
+		}
+		d.buf[pos+int64(i)] = b
+		written++
+	}
+	return written, nil
+}
+
+func validate(t *testing.T, key string, md5value string) {
+	mgr := s3manager.NewDownloader(integration.Session)
+	params := &s3.GetObjectInput{Bucket: bucketName, Key: &key}
+
+	w := newDLWriter(1024 * 1024 * 20)
+	n, err := mgr.Download(w, params)
+	if err != nil {
+		t.Fatalf("expect no error, got %v", err)
+	}
+	if e, a := md5value, fmt.Sprintf("%x", md5.Sum(w.buf[0:n])); e != a {
+		t.Errorf("expect %s md5 value, got %s", e, a)
+	}
+}
+
+func TestUploadConcurrently(t *testing.T) {
+	key := "12mb-1"
+	mgr := s3manager.NewUploader(integration.Session)
+	out, err := mgr.Upload(&s3manager.UploadInput{
+		Bucket: bucketName,
+		Key:    &key,
+		Body:   bytes.NewReader(integBuf12MB),
+	})
+
+	if err != nil {
+		t.Fatalf("expect no error, got %v", err)
+	}
+	if len(out.UploadID) == 0 {
+		t.Errorf("expect upload ID but was empty")
+	}
+
+	re := regexp.MustCompile(`^https?://.+/` + key + `$`)
+	if e, a := re.String(), out.Location; !re.MatchString(a) {
+		t.Errorf("expect %s to match URL regexp %q, did not", e, a)
+	}
+
+	validate(t, key, integMD512MB)
+}
+
+func TestUploadFailCleanup(t *testing.T) {
+	svc := s3.New(integration.Session)
+
+	// Break checksum on 2nd part so it fails
+	part := 0
+	svc.Handlers.Build.PushBack(func(r *request.Request) {
+		if r.Operation.Name == "UploadPart" {
+			if part == 1 {
+				r.HTTPRequest.Header.Set("X-Amz-Content-Sha256", "000")
+			}
+			part++
+		}
+	})
+
+	key := "12mb-leave"
+	mgr := s3manager.NewUploaderWithClient(svc, func(u *s3manager.Uploader) {
+		u.LeavePartsOnError = false
+	})
+	_, err := mgr.Upload(&s3manager.UploadInput{
+		Bucket: bucketName,
+		Key:    &key,
+		Body:   bytes.NewReader(integBuf12MB),
+	})
+	if err == nil {
+		t.Fatalf("expect error, but did not get one")
+	}
+
+	aerr := err.(awserr.Error)
+	if e, a := "MissingRegion", aerr.Code(); strings.Contains(a, e) {
+		t.Errorf("expect %q to not be in error code %q", e, a)
+	}
+
+	uploadID := ""
+	merr := err.(s3manager.MultiUploadFailure)
+	if uploadID = merr.UploadID(); len(uploadID) == 0 {
+		t.Errorf("expect upload ID to not be empty, but was")
+	}
+
+	_, err = svc.ListParts(&s3.ListPartsInput{
+		Bucket: bucketName, Key: &key, UploadId: &uploadID,
+	})
+	if err == nil {
+		t.Errorf("expect error for list parts, but got none")
+	}
+}

vendor/github.com/aws/aws-sdk-go/awstesting/integration/customizations/s3/s3manager/stub.go

@@ -0,0 +1 @@
+package s3manager

vendor/github.com/aws/aws-sdk-go/awstesting/integration/customizations/s3/stub.go

@@ -0,0 +1 @@
+package s3