Switch to using the dep tool and update all the dependencies

This commit is contained in:
Nick Craig-Wood 2017-05-11 15:39:54 +01:00
parent 5135ff73cb
commit 98c2d2c41b
5321 changed files with 4483201 additions and 5922 deletions

View file

@ -0,0 +1,14 @@
# Example
This is an example using the AWS SDK for Go to concatenate two objects together.
We use `UploadPartCopy` which uses an object for a part. Here in this example we have two parts, or in other words
two objects that we want to concatenate together.
# Usage
The example uses the bucket name provided, two keys for each object, and lastly the output key.
```sh
AWS_REGION=<region> go run -tags example concatenateObjects.go <bucket> <key for object 1> <key for object 2> <key for output>
```

View file

@ -0,0 +1,104 @@
// +build example
package main
import (
"log"
"net/url"
"os"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/session"
"github.com/aws/aws-sdk-go/service/s3"
)
type client struct {
s3Client *s3.S3
bucket *string
}
// concatenate will contenate key1's object to key2's object under the key testKey
func (c *client) concatenate(key1, key2, key3 string, uploadID *string) (*string, *string, error) {
// The first part to be uploaded which is represented as part number 1
foo, err := c.s3Client.UploadPartCopy(&s3.UploadPartCopyInput{
Bucket: c.bucket,
CopySource: aws.String(url.QueryEscape(*c.bucket + "/" + key1)),
PartNumber: aws.Int64(1),
Key: &key3,
UploadId: uploadID,
})
if err != nil {
return nil, nil, err
}
// The second part that is going to be appended to the newly created testKey
// object.
bar, err := c.s3Client.UploadPartCopy(&s3.UploadPartCopyInput{
Bucket: c.bucket,
CopySource: aws.String(url.QueryEscape(*c.bucket + "/" + key2)),
PartNumber: aws.Int64(2),
Key: &key3,
UploadId: uploadID,
})
if err != nil {
return nil, nil, err
}
// The ETags are needed to complete the process
return foo.CopyPartResult.ETag, bar.CopyPartResult.ETag, nil
}
func main() {
if len(os.Args) < 4 {
log.Println("USAGE ERROR: AWS_REGION=us-east-1 go run concatenateObjects.go <bucket> <key for object 1> <key for object 2> <key for output>")
return
}
bucket := os.Args[1]
key1 := os.Args[2]
key2 := os.Args[3]
key3 := os.Args[4]
sess := session.New(&aws.Config{})
svc := s3.New(sess)
c := client{svc, &bucket}
// We let the service know that we want to do a multipart upload
output, err := c.s3Client.CreateMultipartUpload(&s3.CreateMultipartUploadInput{
Bucket: &bucket,
Key: &key3,
})
if err != nil {
log.Println("ERROR:", err)
return
}
foo, bar, err := c.concatenate(key1, key2, key3, output.UploadId)
if err != nil {
log.Println("ERROR:", err)
return
}
// We finally complete the multipart upload.
_, err = c.s3Client.CompleteMultipartUpload(&s3.CompleteMultipartUploadInput{
Bucket: &bucket,
Key: &key3,
UploadId: output.UploadId,
MultipartUpload: &s3.CompletedMultipartUpload{
Parts: []*s3.CompletedPart{
{
ETag: foo,
PartNumber: aws.Int64(1),
},
{
ETag: bar,
PartNumber: aws.Int64(2),
},
},
},
})
if err != nil {
log.Println("ERROR:", err)
return
}
}

View file

@ -0,0 +1,27 @@
# Example
This is an example using the AWS SDK for Go to list objects' key in a S3 bucket.
# Usage
The example uses the bucket name provided, and lists all object keys in a bucket.
```sh
go run -tags example listObjects.go <bucket>
```
Output:
```
Page, 0
Object: myKey
Object: mykey.txt
Object: resources/0001/item-01
Object: resources/0001/item-02
Object: resources/0001/item-03
Object: resources/0002/item-01
Object: resources/0002/item-02
Object: resources/0002/item-03
Object: resources/0002/item-04
Object: resources/0002/item-05
```

View file

@ -0,0 +1,43 @@
// +build example
package main
import (
"fmt"
"os"
"github.com/aws/aws-sdk-go/aws/session"
"github.com/aws/aws-sdk-go/service/s3"
)
// Lists all objects in a bucket using pagination
//
// Usage:
// listObjects <bucket>
func main() {
if len(os.Args) < 2 {
fmt.Println("you must specify a bucket")
return
}
sess := session.Must(session.NewSession())
svc := s3.New(sess)
i := 0
err := svc.ListObjectsPages(&s3.ListObjectsInput{
Bucket: &os.Args[1],
}, func(p *s3.ListObjectsOutput, last bool) (shouldContinue bool) {
fmt.Println("Page,", i)
i++
for _, obj := range p.Contents {
fmt.Println("Object:", *obj.Key)
}
return true
})
if err != nil {
fmt.Println("failed to list objects", err)
return
}
}

View file

@ -0,0 +1,13 @@
## Example
This is an example using the AWS SDK for Go concurrently to list the encrypted objects in the S3 buckets owned by an account.
## Usage
The example's `accounts` string slice contains a list of the SharedCredentials profiles which will be used to look up the buckets owned by each profile. Each bucket's objects will be queried.
```
AWS_REGION=us-east-1 go run -tags example listObjectsConcurrentlv.go
```

View file

@ -0,0 +1,236 @@
// +build example
package main
import (
"fmt"
"os"
"sort"
"sync"
"time"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/session"
"github.com/aws/aws-sdk-go/service/s3"
)
func exit(msg ...interface{}) {
fmt.Fprintln(os.Stderr, msg...)
os.Exit(1)
}
// Lists all encrypted objects owned by an account. The `accounts` string
// contains a list of profiles to use.
//
// Usage:
// listObjectsConcurrentlv
func main() {
accounts := []string{"default", "default2", "otherprofile"}
// Spin off a worker for each account to retrieve that account's
bucketCh := make(chan *Bucket, 5)
var wg sync.WaitGroup
for _, acc := range accounts {
wg.Add(1)
go func(acc string) {
defer wg.Done()
sess, err := session.NewSessionWithOptions(session.Options{
Profile: acc,
})
if err != nil {
fmt.Fprintf(os.Stderr, "failed to create session for account, %s, %v\n", acc, err)
return
}
if err = getAccountBuckets(sess, bucketCh, acc); err != nil {
fmt.Fprintf(os.Stderr, "failed to get account %s's bucket info, %v\n", acc, err)
}
}(acc)
}
// Spin off a goroutine which will wait until all account buckets have been collected and
// added to the bucketCh. Close the bucketCh so the for range below will exit once all
// bucket info is printed.
go func() {
wg.Wait()
close(bucketCh)
}()
// Receive from the bucket channel printing the information for each bucket to the console
// when the bucketCh channel is drained.
buckets := []*Bucket{}
for b := range bucketCh {
buckets = append(buckets, b)
}
sortBuckets(buckets)
for _, b := range buckets {
if b.Error != nil {
fmt.Printf("Bucket %s, owned by: %s, failed: %v\n", b.Name, b.Owner, b.Error)
continue
}
encObjs := b.encryptedObjects()
fmt.Printf("Bucket: %s, owned by: %s, total objects: %d, failed objects: %d, encrypted objects: %d\n",
b.Name, b.Owner, len(b.Objects), len(b.ErrObjects), len(encObjs))
if len(encObjs) > 0 {
for _, encObj := range encObjs {
fmt.Printf("\t%s %s:%s/%s\n", encObj.EncryptionType, b.Region, b.Name, encObj.Key)
}
}
}
}
func sortBuckets(buckets []*Bucket) {
s := sortalbeBuckets(buckets)
sort.Sort(s)
}
type sortalbeBuckets []*Bucket
func (s sortalbeBuckets) Len() int { return len(s) }
func (s sortalbeBuckets) Swap(a, b int) { s[a], s[b] = s[b], s[a] }
func (s sortalbeBuckets) Less(a, b int) bool {
if s[a].Owner == s[b].Owner && s[a].Name < s[b].Name {
return true
}
if s[a].Owner < s[b].Owner {
return true
}
return false
}
func getAccountBuckets(sess *session.Session, bucketCh chan<- *Bucket, owner string) error {
svc := s3.New(sess)
buckets, err := listBuckets(svc)
if err != nil {
return fmt.Errorf("failed to list buckets, %v", err)
}
for _, bucket := range buckets {
bucket.Owner = owner
if bucket.Error != nil {
continue
}
bckSvc := s3.New(sess, &aws.Config{
Region: aws.String(bucket.Region),
Credentials: svc.Config.Credentials,
})
bucketDetails(bckSvc, bucket)
bucketCh <- bucket
}
return nil
}
func bucketDetails(svc *s3.S3, bucket *Bucket) {
objs, errObjs, err := listBucketObjects(svc, bucket.Name)
if err != nil {
bucket.Error = err
} else {
bucket.Objects = objs
bucket.ErrObjects = errObjs
}
}
// A Object provides details of an S3 object
type Object struct {
Bucket string
Key string
Encrypted bool
EncryptionType string
}
// An ErrObject provides details of the error occurred retrieving
// an object's status.
type ErrObject struct {
Bucket string
Key string
Error error
}
// A Bucket provides details about a bucket and its objects
type Bucket struct {
Owner string
Name string
CreationDate time.Time
Region string
Objects []Object
Error error
ErrObjects []ErrObject
}
func (b *Bucket) encryptedObjects() []Object {
encObjs := []Object{}
for _, obj := range b.Objects {
if obj.Encrypted {
encObjs = append(encObjs, obj)
}
}
return encObjs
}
func listBuckets(svc *s3.S3) ([]*Bucket, error) {
res, err := svc.ListBuckets(&s3.ListBucketsInput{})
if err != nil {
return nil, err
}
buckets := make([]*Bucket, len(res.Buckets))
for i, b := range res.Buckets {
buckets[i] = &Bucket{
Name: *b.Name,
CreationDate: *b.CreationDate,
}
locRes, err := svc.GetBucketLocation(&s3.GetBucketLocationInput{
Bucket: b.Name,
})
if err != nil {
buckets[i].Error = err
continue
}
if locRes.LocationConstraint == nil {
buckets[i].Region = "us-east-1"
} else {
buckets[i].Region = *locRes.LocationConstraint
}
}
return buckets, nil
}
func listBucketObjects(svc *s3.S3, bucket string) ([]Object, []ErrObject, error) {
listRes, err := svc.ListObjects(&s3.ListObjectsInput{
Bucket: &bucket,
})
if err != nil {
return nil, nil, err
}
objs := make([]Object, 0, len(listRes.Contents))
errObjs := []ErrObject{}
for _, listObj := range listRes.Contents {
objData, err := svc.HeadObject(&s3.HeadObjectInput{
Bucket: &bucket,
Key: listObj.Key,
})
if err != nil {
errObjs = append(errObjs, ErrObject{Bucket: bucket, Key: *listObj.Key, Error: err})
continue
}
obj := Object{Bucket: bucket, Key: *listObj.Key}
if objData.ServerSideEncryption != nil {
obj.Encrypted = true
obj.EncryptionType = *objData.ServerSideEncryption
}
objs = append(objs, obj)
}
return objs, errObjs, nil
}

View file

@ -0,0 +1,40 @@
# Example
putObjectAcl is an example using the AWS SDK for Go to put an ACL on an S3 object.
# Usage
```sh
putBucketAcl <params>
-region <region> // required
-bucket <bucket> // required
-key <key> // required
-owner-name <owner-name>
-owner-id <owner-id>
-grantee-type <some type> // required
-uri <uri to group>
-email <email address>
-user-id <user-id>
-display-name <display name>
```
```sh
go run -tags example putObjectAcl.go
-bucket <bucket>
-key <key>
-owner-name <name>
-owner-id <id>
-grantee-type <some type>
-user-id <user-id>
```
Depending on the type is used depends on which of the three, `uri`, `email`, or `user-id`, needs to be used.
* `s3.TypeCanonicalUser`: `user-id` or `display-name` must be used
* `s3.TypeAmazonCustomerByEmail`: `email` must be used
* `s3.TypeGroup`: `uri` must be used
Output:
```
success {
} nil
```

View file

@ -0,0 +1,91 @@
// +build example
package main
import (
"flag"
"fmt"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/session"
"github.com/aws/aws-sdk-go/service/s3"
)
// Put an ACL on an S3 object
//
// Usage:
// putBucketAcl <params>
// -region <region> // required
// -bucket <bucket> // required
// -key <key> // required
// -owner-name <owner-name>
// -owner-id <owner-id>
// -grantee-type <some type> // required
// -uri <uri to group>
// -email <email address>
// -user-id <user-id>
func main() {
regionPtr := flag.String("region", "", "region of your request")
bucketPtr := flag.String("bucket", "", "name of your bucket")
keyPtr := flag.String("key", "", "of your object")
ownerNamePtr := flag.String("owner-name", "", "of your request")
ownerIDPtr := flag.String("owner-id", "", "of your request")
granteeTypePtr := flag.String("grantee-type", "", "of your request")
uriPtr := flag.String("uri", "", "of your grantee type")
emailPtr := flag.String("email", "", "of your grantee type")
userPtr := flag.String("user-id", "", "of your grantee type")
displayNamePtr := flag.String("display-name", "", "of your grantee type")
flag.Parse()
// Based off the type, fields must be excluded.
switch *granteeTypePtr {
case s3.TypeCanonicalUser:
emailPtr, uriPtr = nil, nil
if *displayNamePtr == "" {
displayNamePtr = nil
}
if *userPtr == "" {
userPtr = nil
}
case s3.TypeAmazonCustomerByEmail:
uriPtr, userPtr = nil, nil
case s3.TypeGroup:
emailPtr, userPtr = nil, nil
}
sess := session.Must(session.NewSession(&aws.Config{
Region: regionPtr,
}))
svc := s3.New(sess)
resp, err := svc.PutObjectAcl(&s3.PutObjectAclInput{
Bucket: bucketPtr,
Key: keyPtr,
AccessControlPolicy: &s3.AccessControlPolicy{
Owner: &s3.Owner{
DisplayName: ownerNamePtr,
ID: ownerIDPtr,
},
Grants: []*s3.Grant{
{
Grantee: &s3.Grantee{
Type: granteeTypePtr,
DisplayName: displayNamePtr,
URI: uriPtr,
EmailAddress: emailPtr,
ID: userPtr,
},
Permission: aws.String(s3.BucketLogsPermissionFullControl),
},
},
},
})
if err != nil {
fmt.Println("failed", err)
} else {
fmt.Println("success", resp)
}
}