vendor: update all dependencies

This commit is contained in:
Nick Craig-Wood 2018-06-17 17:59:12 +01:00
parent 3f0789e2db
commit 08021c4636
2474 changed files with 435818 additions and 282709 deletions

View file

@ -3,14 +3,20 @@
package s3
import (
"bytes"
"fmt"
"io"
"sync"
"sync/atomic"
"time"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/awsutil"
"github.com/aws/aws-sdk-go/aws/request"
"github.com/aws/aws-sdk-go/private/protocol"
"github.com/aws/aws-sdk-go/private/protocol/eventstream"
"github.com/aws/aws-sdk-go/private/protocol/eventstream/eventstreamapi"
"github.com/aws/aws-sdk-go/private/protocol/rest"
"github.com/aws/aws-sdk-go/private/protocol/restxml"
)
@ -6017,6 +6023,87 @@ func (c *S3) RestoreObjectWithContext(ctx aws.Context, input *RestoreObjectInput
return out, req.Send()
}
const opSelectObjectContent = "SelectObjectContent"
// SelectObjectContentRequest generates a "aws/request.Request" representing the
// client's request for the SelectObjectContent operation. The "output" return
// value will be populated with the request's response once the request completes
// successfuly.
//
// Use "Send" method on the returned Request to send the API call to the service.
// the "output" return value is not valid until after Send returns without error.
//
// See SelectObjectContent for more information on using the SelectObjectContent
// API call, and error handling.
//
// This method is useful when you want to inject custom logic or configuration
// into the SDK's request lifecycle. Such as custom headers, or retry logic.
//
//
// // Example sending a request using the SelectObjectContentRequest method.
// req, resp := client.SelectObjectContentRequest(params)
//
// err := req.Send()
// if err == nil { // resp is now filled
// fmt.Println(resp)
// }
//
// See also, https://docs.aws.amazon.com/goto/WebAPI/s3-2006-03-01/SelectObjectContent
func (c *S3) SelectObjectContentRequest(input *SelectObjectContentInput) (req *request.Request, output *SelectObjectContentOutput) {
op := &request.Operation{
Name: opSelectObjectContent,
HTTPMethod: "POST",
HTTPPath: "/{Bucket}/{Key+}?select&select-type=2",
}
if input == nil {
input = &SelectObjectContentInput{}
}
output = &SelectObjectContentOutput{}
req = c.newRequest(op, input, output)
req.Handlers.Unmarshal.Swap(restxml.UnmarshalHandler.Name, rest.UnmarshalHandler)
req.Handlers.Unmarshal.PushBack(output.runEventStreamLoop)
return
}
// SelectObjectContent API operation for Amazon Simple Storage Service.
//
// This operation filters the contents of an Amazon S3 object based on a simple
// Structured Query Language (SQL) statement. In the request, along with the
// SQL expression, you must also specify a data serialization format (JSON or
// CSV) of the object. Amazon S3 uses this to parse object data into records,
// and returns only records that match the specified SQL expression. You must
// also specify the data serialization format for the response.
//
// Returns awserr.Error for service API and SDK errors. Use runtime type assertions
// with awserr.Error's Code and Message methods to get detailed information about
// the error.
//
// See the AWS API reference guide for Amazon Simple Storage Service's
// API operation SelectObjectContent for usage and error information.
// See also, https://docs.aws.amazon.com/goto/WebAPI/s3-2006-03-01/SelectObjectContent
func (c *S3) SelectObjectContent(input *SelectObjectContentInput) (*SelectObjectContentOutput, error) {
req, out := c.SelectObjectContentRequest(input)
return out, req.Send()
}
// SelectObjectContentWithContext is the same as SelectObjectContent with the addition of
// the ability to pass a context and additional request options.
//
// See SelectObjectContent for details on how to use this API operation.
//
// The context must be non-nil and will be used for request cancellation. If
// the context is nil a panic will occur. In the future the SDK may create
// sub-contexts for http.Requests. See https://golang.org/pkg/context/
// for more information on using Contexts.
func (c *S3) SelectObjectContentWithContext(ctx aws.Context, input *SelectObjectContentInput, opts ...request.Option) (*SelectObjectContentOutput, error) {
req, out := c.SelectObjectContentRequest(input)
req.SetContext(ctx)
req.ApplyOptions(opts...)
return out, req.Send()
}
const opUploadPart = "UploadPart"
// UploadPartRequest generates a "aws/request.Request" representing the
@ -7474,6 +7561,32 @@ func (s *Condition) SetKeyPrefixEquals(v string) *Condition {
return s
}
type ContinuationEvent struct {
_ struct{} `type:"structure"`
}
// String returns the string representation
func (s ContinuationEvent) String() string {
return awsutil.Prettify(s)
}
// GoString returns the string representation
func (s ContinuationEvent) GoString() string {
return s.String()
}
// The ContinuationEvent is and event in the SelectObjectContentEventStream group of events.
func (s *ContinuationEvent) eventSelectObjectContentEventStream() {}
// UnmarshalEvent unmarshals the EventStream Message into the ContinuationEvent value.
// This method is only used internally within the SDK's EventStream handling.
func (s *ContinuationEvent) UnmarshalEvent(
payloadUnmarshaler protocol.PayloadUnmarshaler,
msg eventstream.Message,
) error {
return nil
}
type CopyObjectInput struct {
_ struct{} `type:"structure"`
@ -9919,6 +10032,32 @@ func (s *EncryptionConfiguration) SetReplicaKmsKeyID(v string) *EncryptionConfig
return s
}
type EndEvent struct {
_ struct{} `type:"structure"`
}
// String returns the string representation
func (s EndEvent) String() string {
return awsutil.Prettify(s)
}
// GoString returns the string representation
func (s EndEvent) GoString() string {
return s.String()
}
// The EndEvent is and event in the SelectObjectContentEventStream group of events.
func (s *EndEvent) eventSelectObjectContentEventStream() {}
// UnmarshalEvent unmarshals the EventStream Message into the EndEvent value.
// This method is only used internally within the SDK's EventStream handling.
func (s *EndEvent) UnmarshalEvent(
payloadUnmarshaler protocol.PayloadUnmarshaler,
msg eventstream.Message,
) error {
return nil
}
type Error struct {
_ struct{} `type:"structure"`
@ -16380,6 +16519,87 @@ func (s *Part) SetSize(v int64) *Part {
return s
}
type Progress struct {
_ struct{} `type:"structure"`
// Current number of uncompressed object bytes processed.
BytesProcessed *int64 `type:"long"`
// Current number of bytes of records payload data returned.
BytesReturned *int64 `type:"long"`
// Current number of object bytes scanned.
BytesScanned *int64 `type:"long"`
}
// String returns the string representation
func (s Progress) String() string {
return awsutil.Prettify(s)
}
// GoString returns the string representation
func (s Progress) GoString() string {
return s.String()
}
// SetBytesProcessed sets the BytesProcessed field's value.
func (s *Progress) SetBytesProcessed(v int64) *Progress {
s.BytesProcessed = &v
return s
}
// SetBytesReturned sets the BytesReturned field's value.
func (s *Progress) SetBytesReturned(v int64) *Progress {
s.BytesReturned = &v
return s
}
// SetBytesScanned sets the BytesScanned field's value.
func (s *Progress) SetBytesScanned(v int64) *Progress {
s.BytesScanned = &v
return s
}
type ProgressEvent struct {
_ struct{} `type:"structure" payload:"Details"`
// The Progress event details.
Details *Progress `locationName:"Details" type:"structure"`
}
// String returns the string representation
func (s ProgressEvent) String() string {
return awsutil.Prettify(s)
}
// GoString returns the string representation
func (s ProgressEvent) GoString() string {
return s.String()
}
// SetDetails sets the Details field's value.
func (s *ProgressEvent) SetDetails(v *Progress) *ProgressEvent {
s.Details = v
return s
}
// The ProgressEvent is and event in the SelectObjectContentEventStream group of events.
func (s *ProgressEvent) eventSelectObjectContentEventStream() {}
// UnmarshalEvent unmarshals the EventStream Message into the ProgressEvent value.
// This method is only used internally within the SDK's EventStream handling.
func (s *ProgressEvent) UnmarshalEvent(
payloadUnmarshaler protocol.PayloadUnmarshaler,
msg eventstream.Message,
) error {
if err := payloadUnmarshaler.UnmarshalPayload(
bytes.NewReader(msg.Payload), s,
); err != nil {
return fmt.Errorf("failed to unmarshal payload, %v", err)
}
return nil
}
type PutBucketAccelerateConfigurationInput struct {
_ struct{} `type:"structure" payload:"AccelerateConfiguration"`
@ -18622,6 +18842,45 @@ func (s *QueueConfigurationDeprecated) SetQueue(v string) *QueueConfigurationDep
return s
}
type RecordsEvent struct {
_ struct{} `type:"structure" payload:"Payload"`
// The byte array of partial, one or more result records.
//
// Payload is automatically base64 encoded/decoded by the SDK.
Payload []byte `type:"blob"`
}
// String returns the string representation
func (s RecordsEvent) String() string {
return awsutil.Prettify(s)
}
// GoString returns the string representation
func (s RecordsEvent) GoString() string {
return s.String()
}
// SetPayload sets the Payload field's value.
func (s *RecordsEvent) SetPayload(v []byte) *RecordsEvent {
s.Payload = v
return s
}
// The RecordsEvent is and event in the SelectObjectContentEventStream group of events.
func (s *RecordsEvent) eventSelectObjectContentEventStream() {}
// UnmarshalEvent unmarshals the EventStream Message into the RecordsEvent value.
// This method is only used internally within the SDK's EventStream handling.
func (s *RecordsEvent) UnmarshalEvent(
payloadUnmarshaler protocol.PayloadUnmarshaler,
msg eventstream.Message,
) error {
s.Payload = make([]byte, len(msg.Payload))
copy(s.Payload, msg.Payload)
return nil
}
type Redirect struct {
_ struct{} `type:"structure"`
@ -18939,6 +19198,30 @@ func (s *RequestPaymentConfiguration) SetPayer(v string) *RequestPaymentConfigur
return s
}
type RequestProgress struct {
_ struct{} `type:"structure"`
// Specifies whether periodic QueryProgress frames should be sent. Valid values:
// TRUE, FALSE. Default value: FALSE.
Enabled *bool `type:"boolean"`
}
// String returns the string representation
func (s RequestProgress) String() string {
return awsutil.Prettify(s)
}
// GoString returns the string representation
func (s RequestProgress) GoString() string {
return s.String()
}
// SetEnabled sets the Enabled field's value.
func (s *RequestProgress) SetEnabled(v bool) *RequestProgress {
s.Enabled = &v
return s
}
type RestoreObjectInput struct {
_ struct{} `type:"structure" payload:"RestoreRequest"`
@ -19392,6 +19675,436 @@ func (s SSES3) GoString() string {
return s.String()
}
// SelectObjectContentEventStream provides handling of EventStreams for
// the SelectObjectContent API.
//
// Use this type to receive SelectObjectContentEventStream events. The events
// can be read from the Events channel member.
//
// The events that can be received are:
//
// * ContinuationEvent
// * EndEvent
// * ProgressEvent
// * RecordsEvent
// * StatsEvent
type SelectObjectContentEventStream struct {
// Reader is the EventStream reader for the SelectObjectContentEventStream
// events. This value is automatically set by the SDK when the API call is made
// Use this member when unit testing your code with the SDK to mock out the
// EventStream Reader.
//
// Must not be nil.
Reader SelectObjectContentEventStreamReader
// StreamCloser is the io.Closer for the EventStream connection. For HTTP
// EventStream this is the response Body. The stream will be closed when
// the Close method of the EventStream is called.
StreamCloser io.Closer
}
// Close closes the EventStream. This will also cause the Events channel to be
// closed. You can use the closing of the Events channel to terminate your
// application's read from the API's EventStream.
//
// Will close the underlying EventStream reader. For EventStream over HTTP
// connection this will also close the HTTP connection.
//
// Close must be called when done using the EventStream API. Not calling Close
// may result in resource leaks.
func (es *SelectObjectContentEventStream) Close() (err error) {
es.Reader.Close()
return es.Err()
}
// Err returns any error that occurred while reading EventStream Events from
// the service API's response. Returns nil if there were no errors.
func (es *SelectObjectContentEventStream) Err() error {
if err := es.Reader.Err(); err != nil {
return err
}
es.StreamCloser.Close()
return nil
}
// Events returns a channel to read EventStream Events from the
// SelectObjectContent API.
//
// These events are:
//
// * ContinuationEvent
// * EndEvent
// * ProgressEvent
// * RecordsEvent
// * StatsEvent
func (es *SelectObjectContentEventStream) Events() <-chan SelectObjectContentEventStreamEvent {
return es.Reader.Events()
}
// SelectObjectContentEventStreamEvent groups together all EventStream
// events read from the SelectObjectContent API.
//
// These events are:
//
// * ContinuationEvent
// * EndEvent
// * ProgressEvent
// * RecordsEvent
// * StatsEvent
type SelectObjectContentEventStreamEvent interface {
eventSelectObjectContentEventStream()
}
// SelectObjectContentEventStreamReader provides the interface for reading EventStream
// Events from the SelectObjectContent API. The
// default implementation for this interface will be SelectObjectContentEventStream.
//
// The reader's Close method must allow multiple concurrent calls.
//
// These events are:
//
// * ContinuationEvent
// * EndEvent
// * ProgressEvent
// * RecordsEvent
// * StatsEvent
type SelectObjectContentEventStreamReader interface {
// Returns a channel of events as they are read from the event stream.
Events() <-chan SelectObjectContentEventStreamEvent
// Close will close the underlying event stream reader. For event stream over
// HTTP this will also close the HTTP connection.
Close() error
// Returns any error that has occured while reading from the event stream.
Err() error
}
type readSelectObjectContentEventStream struct {
eventReader *eventstreamapi.EventReader
stream chan SelectObjectContentEventStreamEvent
errVal atomic.Value
done chan struct{}
closeOnce sync.Once
}
func newReadSelectObjectContentEventStream(
reader io.ReadCloser,
unmarshalers request.HandlerList,
logger aws.Logger,
logLevel aws.LogLevelType,
) *readSelectObjectContentEventStream {
r := &readSelectObjectContentEventStream{
stream: make(chan SelectObjectContentEventStreamEvent),
done: make(chan struct{}),
}
r.eventReader = eventstreamapi.NewEventReader(
reader,
protocol.HandlerPayloadUnmarshal{
Unmarshalers: unmarshalers,
},
r.unmarshalerForEventType,
)
r.eventReader.UseLogger(logger, logLevel)
return r
}
// Close will close the underlying event stream reader. For EventStream over
// HTTP this will also close the HTTP connection.
func (r *readSelectObjectContentEventStream) Close() error {
r.closeOnce.Do(r.safeClose)
return r.Err()
}
func (r *readSelectObjectContentEventStream) safeClose() {
close(r.done)
err := r.eventReader.Close()
if err != nil {
r.errVal.Store(err)
}
}
func (r *readSelectObjectContentEventStream) Err() error {
if v := r.errVal.Load(); v != nil {
return v.(error)
}
return nil
}
func (r *readSelectObjectContentEventStream) Events() <-chan SelectObjectContentEventStreamEvent {
return r.stream
}
func (r *readSelectObjectContentEventStream) readEventStream() {
defer close(r.stream)
for {
event, err := r.eventReader.ReadEvent()
if err != nil {
if err == io.EOF {
return
}
select {
case <-r.done:
// If closed already ignore the error
return
default:
}
r.errVal.Store(err)
return
}
select {
case r.stream <- event.(SelectObjectContentEventStreamEvent):
case <-r.done:
return
}
}
}
func (r *readSelectObjectContentEventStream) unmarshalerForEventType(
eventType string,
) (eventstreamapi.Unmarshaler, error) {
switch eventType {
case "Cont":
return &ContinuationEvent{}, nil
case "End":
return &EndEvent{}, nil
case "Progress":
return &ProgressEvent{}, nil
case "Records":
return &RecordsEvent{}, nil
case "Stats":
return &StatsEvent{}, nil
default:
return nil, fmt.Errorf(
"unknown event type name, %s, for SelectObjectContentEventStream", eventType)
}
}
// Request to filter the contents of an Amazon S3 object based on a simple Structured
// Query Language (SQL) statement. In the request, along with the SQL expression,
// you must also specify a data serialization format (JSON or CSV) of the object.
// Amazon S3 uses this to parse object data into records, and returns only records
// that match the specified SQL expression. You must also specify the data serialization
// format for the response. For more information, go to S3Select API Documentation
// (https://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectSELECTContent.html)
type SelectObjectContentInput struct {
_ struct{} `locationName:"SelectObjectContentRequest" type:"structure" xmlURI:"http://s3.amazonaws.com/doc/2006-03-01/"`
// The S3 Bucket.
//
// Bucket is a required field
Bucket *string `location:"uri" locationName:"Bucket" type:"string" required:"true"`
// The expression that is used to query the object.
//
// Expression is a required field
Expression *string `type:"string" required:"true"`
// The type of the provided expression (e.g., SQL).
//
// ExpressionType is a required field
ExpressionType *string `type:"string" required:"true" enum:"ExpressionType"`
// Describes the format of the data in the object that is being queried.
//
// InputSerialization is a required field
InputSerialization *InputSerialization `type:"structure" required:"true"`
// The Object Key.
//
// Key is a required field
Key *string `location:"uri" locationName:"Key" min:"1" type:"string" required:"true"`
// Describes the format of the data that you want Amazon S3 to return in response.
//
// OutputSerialization is a required field
OutputSerialization *OutputSerialization `type:"structure" required:"true"`
// Specifies if periodic request progress information should be enabled.
RequestProgress *RequestProgress `type:"structure"`
// The SSE Algorithm used to encrypt the object. For more information, go to
// Server-Side Encryption (Using Customer-Provided Encryption Keys (https://docs.aws.amazon.com/AmazonS3/latest/dev/ServerSideEncryptionCustomerKeys.html)
SSECustomerAlgorithm *string `location:"header" locationName:"x-amz-server-side-encryption-customer-algorithm" type:"string"`
// The SSE Customer Key. For more information, go to Server-Side Encryption
// (Using Customer-Provided Encryption Keys (https://docs.aws.amazon.com/AmazonS3/latest/dev/ServerSideEncryptionCustomerKeys.html)
SSECustomerKey *string `location:"header" locationName:"x-amz-server-side-encryption-customer-key" type:"string"`
// The SSE Customer Key MD5. For more information, go to Server-Side Encryption
// (Using Customer-Provided Encryption Keys (https://docs.aws.amazon.com/AmazonS3/latest/dev/ServerSideEncryptionCustomerKeys.html)
SSECustomerKeyMD5 *string `location:"header" locationName:"x-amz-server-side-encryption-customer-key-MD5" type:"string"`
}
// String returns the string representation
func (s SelectObjectContentInput) String() string {
return awsutil.Prettify(s)
}
// GoString returns the string representation
func (s SelectObjectContentInput) GoString() string {
return s.String()
}
// Validate inspects the fields of the type to determine if they are valid.
func (s *SelectObjectContentInput) Validate() error {
invalidParams := request.ErrInvalidParams{Context: "SelectObjectContentInput"}
if s.Bucket == nil {
invalidParams.Add(request.NewErrParamRequired("Bucket"))
}
if s.Expression == nil {
invalidParams.Add(request.NewErrParamRequired("Expression"))
}
if s.ExpressionType == nil {
invalidParams.Add(request.NewErrParamRequired("ExpressionType"))
}
if s.InputSerialization == nil {
invalidParams.Add(request.NewErrParamRequired("InputSerialization"))
}
if s.Key == nil {
invalidParams.Add(request.NewErrParamRequired("Key"))
}
if s.Key != nil && len(*s.Key) < 1 {
invalidParams.Add(request.NewErrParamMinLen("Key", 1))
}
if s.OutputSerialization == nil {
invalidParams.Add(request.NewErrParamRequired("OutputSerialization"))
}
if invalidParams.Len() > 0 {
return invalidParams
}
return nil
}
// SetBucket sets the Bucket field's value.
func (s *SelectObjectContentInput) SetBucket(v string) *SelectObjectContentInput {
s.Bucket = &v
return s
}
func (s *SelectObjectContentInput) getBucket() (v string) {
if s.Bucket == nil {
return v
}
return *s.Bucket
}
// SetExpression sets the Expression field's value.
func (s *SelectObjectContentInput) SetExpression(v string) *SelectObjectContentInput {
s.Expression = &v
return s
}
// SetExpressionType sets the ExpressionType field's value.
func (s *SelectObjectContentInput) SetExpressionType(v string) *SelectObjectContentInput {
s.ExpressionType = &v
return s
}
// SetInputSerialization sets the InputSerialization field's value.
func (s *SelectObjectContentInput) SetInputSerialization(v *InputSerialization) *SelectObjectContentInput {
s.InputSerialization = v
return s
}
// SetKey sets the Key field's value.
func (s *SelectObjectContentInput) SetKey(v string) *SelectObjectContentInput {
s.Key = &v
return s
}
// SetOutputSerialization sets the OutputSerialization field's value.
func (s *SelectObjectContentInput) SetOutputSerialization(v *OutputSerialization) *SelectObjectContentInput {
s.OutputSerialization = v
return s
}
// SetRequestProgress sets the RequestProgress field's value.
func (s *SelectObjectContentInput) SetRequestProgress(v *RequestProgress) *SelectObjectContentInput {
s.RequestProgress = v
return s
}
// SetSSECustomerAlgorithm sets the SSECustomerAlgorithm field's value.
func (s *SelectObjectContentInput) SetSSECustomerAlgorithm(v string) *SelectObjectContentInput {
s.SSECustomerAlgorithm = &v
return s
}
// SetSSECustomerKey sets the SSECustomerKey field's value.
func (s *SelectObjectContentInput) SetSSECustomerKey(v string) *SelectObjectContentInput {
s.SSECustomerKey = &v
return s
}
func (s *SelectObjectContentInput) getSSECustomerKey() (v string) {
if s.SSECustomerKey == nil {
return v
}
return *s.SSECustomerKey
}
// SetSSECustomerKeyMD5 sets the SSECustomerKeyMD5 field's value.
func (s *SelectObjectContentInput) SetSSECustomerKeyMD5(v string) *SelectObjectContentInput {
s.SSECustomerKeyMD5 = &v
return s
}
type SelectObjectContentOutput struct {
_ struct{} `type:"structure" payload:"Payload"`
// Use EventStream to use the API's stream.
EventStream *SelectObjectContentEventStream `type:"structure"`
}
// String returns the string representation
func (s SelectObjectContentOutput) String() string {
return awsutil.Prettify(s)
}
// GoString returns the string representation
func (s SelectObjectContentOutput) GoString() string {
return s.String()
}
// SetEventStream sets the EventStream field's value.
func (s *SelectObjectContentOutput) SetEventStream(v *SelectObjectContentEventStream) *SelectObjectContentOutput {
s.EventStream = v
return s
}
func (s *SelectObjectContentOutput) runEventStreamLoop(r *request.Request) {
if r.Error != nil {
return
}
reader := newReadSelectObjectContentEventStream(
r.HTTPResponse.Body,
r.Handlers.UnmarshalStream,
r.Config.Logger,
r.Config.LogLevel.Value(),
)
go reader.readEventStream()
eventStream := &SelectObjectContentEventStream{
StreamCloser: r.HTTPResponse.Body,
Reader: reader,
}
s.EventStream = eventStream
}
// Describes the parameters for Select job types.
type SelectParameters struct {
_ struct{} `type:"structure"`
@ -19696,6 +20409,87 @@ func (s *SseKmsEncryptedObjects) SetStatus(v string) *SseKmsEncryptedObjects {
return s
}
type Stats struct {
_ struct{} `type:"structure"`
// Total number of uncompressed object bytes processed.
BytesProcessed *int64 `type:"long"`
// Total number of bytes of records payload data returned.
BytesReturned *int64 `type:"long"`
// Total number of object bytes scanned.
BytesScanned *int64 `type:"long"`
}
// String returns the string representation
func (s Stats) String() string {
return awsutil.Prettify(s)
}
// GoString returns the string representation
func (s Stats) GoString() string {
return s.String()
}
// SetBytesProcessed sets the BytesProcessed field's value.
func (s *Stats) SetBytesProcessed(v int64) *Stats {
s.BytesProcessed = &v
return s
}
// SetBytesReturned sets the BytesReturned field's value.
func (s *Stats) SetBytesReturned(v int64) *Stats {
s.BytesReturned = &v
return s
}
// SetBytesScanned sets the BytesScanned field's value.
func (s *Stats) SetBytesScanned(v int64) *Stats {
s.BytesScanned = &v
return s
}
type StatsEvent struct {
_ struct{} `type:"structure" payload:"Details"`
// The Stats event details.
Details *Stats `locationName:"Details" type:"structure"`
}
// String returns the string representation
func (s StatsEvent) String() string {
return awsutil.Prettify(s)
}
// GoString returns the string representation
func (s StatsEvent) GoString() string {
return s.String()
}
// SetDetails sets the Details field's value.
func (s *StatsEvent) SetDetails(v *Stats) *StatsEvent {
s.Details = v
return s
}
// The StatsEvent is and event in the SelectObjectContentEventStream group of events.
func (s *StatsEvent) eventSelectObjectContentEventStream() {}
// UnmarshalEvent unmarshals the EventStream Message into the StatsEvent value.
// This method is only used internally within the SDK's EventStream handling.
func (s *StatsEvent) UnmarshalEvent(
payloadUnmarshaler protocol.PayloadUnmarshaler,
msg eventstream.Message,
) error {
if err := payloadUnmarshaler.UnmarshalPayload(
bytes.NewReader(msg.Payload), s,
); err != nil {
return fmt.Errorf("failed to unmarshal payload, %v", err)
}
return nil
}
type StorageClassAnalysis struct {
_ struct{} `type:"structure"`

View file

@ -0,0 +1,297 @@
package s3
import (
"bytes"
"encoding/csv"
"fmt"
"io"
"io/ioutil"
"net/http"
"os"
"reflect"
"testing"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/corehandlers"
"github.com/aws/aws-sdk-go/aws/request"
"github.com/aws/aws-sdk-go/aws/session"
"github.com/aws/aws-sdk-go/awstesting/unit"
"github.com/aws/aws-sdk-go/private/protocol"
"github.com/aws/aws-sdk-go/private/protocol/eventstream"
"github.com/aws/aws-sdk-go/private/protocol/eventstream/eventstreamapi"
"github.com/aws/aws-sdk-go/private/protocol/eventstream/eventstreamtest"
"github.com/aws/aws-sdk-go/private/protocol/restxml"
)
func mockSelectObjectContentEventStreamsOutputEvents() ([]SelectObjectContentEventStreamEvent, []eventstream.Message) {
expectEvents := []SelectObjectContentEventStreamEvent{
&ContinuationEvent{},
&EndEvent{},
&ProgressEvent{
Details: &Progress{
BytesProcessed: aws.Int64(123),
BytesReturned: aws.Int64(456),
BytesScanned: aws.Int64(789),
},
},
&RecordsEvent{
Payload: []byte("abc123"),
},
&StatsEvent{
Details: &Stats{
BytesProcessed: aws.Int64(123),
BytesReturned: aws.Int64(456),
BytesScanned: aws.Int64(789),
},
},
}
var marshalers request.HandlerList
marshalers.PushBackNamed(restxml.BuildHandler)
payloadMarshaler := protocol.HandlerPayloadMarshal{
Marshalers: marshalers,
}
eventMsgs := []eventstream.Message{
{
Headers: eventstream.Headers{
eventstreamtest.EventMessageTypeHeader,
{
Name: eventstreamapi.EventTypeHeader,
Value: eventstream.StringValue("Cont"),
},
},
},
{
Headers: eventstream.Headers{
eventstreamtest.EventMessageTypeHeader,
{
Name: eventstreamapi.EventTypeHeader,
Value: eventstream.StringValue("End"),
},
},
},
{
Headers: eventstream.Headers{
eventstreamtest.EventMessageTypeHeader,
{
Name: eventstreamapi.EventTypeHeader,
Value: eventstream.StringValue("Progress"),
},
},
Payload: eventstreamtest.MarshalEventPayload(payloadMarshaler, expectEvents[2]),
},
{
Headers: eventstream.Headers{
eventstreamtest.EventMessageTypeHeader,
{
Name: eventstreamapi.EventTypeHeader,
Value: eventstream.StringValue("Records"),
},
},
Payload: eventstreamtest.MarshalEventPayload(payloadMarshaler, expectEvents[3]),
},
{
Headers: eventstream.Headers{
eventstreamtest.EventMessageTypeHeader,
{
Name: eventstreamapi.EventTypeHeader,
Value: eventstream.StringValue("Stats"),
},
},
Payload: eventstreamtest.MarshalEventPayload(payloadMarshaler, expectEvents[4]),
},
}
return expectEvents, eventMsgs
}
func TestSelectObjectContentEventStream(t *testing.T) {
expectEvents, eventMsgs := mockSelectObjectContentEventStreamsOutputEvents()
sess, cleanupFn, err := eventstreamtest.SetupEventStreamSession(t,
eventstreamtest.ServeEventStream{
T: t,
Events: eventMsgs,
},
false,
)
if err != nil {
t.Fatalf("expect no error, %v", err)
}
defer cleanupFn()
svc := New(sess)
params := &SelectObjectContentInput{}
resp, err := svc.SelectObjectContent(params)
if err != nil {
t.Fatalf("expect no error got, %v", err)
}
defer resp.EventStream.Close()
var i int
for event := range resp.EventStream.Events() {
if event == nil {
t.Errorf("%d, expect event, got nil", i)
}
if e, a := expectEvents[i], event; !reflect.DeepEqual(e, a) {
t.Errorf("%d, expect %T %v, got %T %v", i, e, e, a, a)
}
i++
}
if err := resp.EventStream.Err(); err != nil {
t.Errorf("expect no error, %v", err)
}
}
func TestSelectObjectContentEventStream_Close(t *testing.T) {
_, eventMsgs := mockSelectObjectContentEventStreamsOutputEvents()
sess, cleanupFn, err := eventstreamtest.SetupEventStreamSession(t,
eventstreamtest.ServeEventStream{
T: t,
Events: eventMsgs,
},
false,
)
if err != nil {
t.Fatalf("expect no error, %v", err)
}
defer cleanupFn()
svc := New(sess)
params := &SelectObjectContentInput{}
resp, err := svc.SelectObjectContent(params)
if err != nil {
t.Fatalf("expect no error got, %v", err)
}
resp.EventStream.Close()
if err := resp.EventStream.Err(); err != nil {
t.Errorf("expect no error, %v", err)
}
}
func ExampleSelectObjectContentEventStream() {
sess := session.Must(session.NewSession())
svc := New(sess)
/*
Example myObjectKey CSV content:
name,number
gopher,0
ᵷodɥǝɹ,1
*/
// Make the Select Object Content API request using the object uploaded.
resp, err := svc.SelectObjectContent(&SelectObjectContentInput{
Bucket: aws.String("myBucket"),
Key: aws.String("myObjectKey"),
Expression: aws.String("SELECT name FROM S3Object WHERE cast(number as int) < 1"),
ExpressionType: aws.String(ExpressionTypeSql),
InputSerialization: &InputSerialization{
CSV: &CSVInput{
FileHeaderInfo: aws.String(FileHeaderInfoUse),
},
},
OutputSerialization: &OutputSerialization{
CSV: &CSVOutput{},
},
})
if err != nil {
fmt.Fprintf(os.Stderr, "failed making API request, %v\n", err)
return
}
defer resp.EventStream.Close()
results, resultWriter := io.Pipe()
go func() {
defer resultWriter.Close()
for event := range resp.EventStream.Events() {
switch e := event.(type) {
case *RecordsEvent:
resultWriter.Write(e.Payload)
case *StatsEvent:
fmt.Printf("Processed %d bytes\n", e.Details.BytesProcessed)
}
}
}()
// Printout the results
resReader := csv.NewReader(results)
for {
record, err := resReader.Read()
if err == io.EOF {
break
}
fmt.Println(record)
}
if err := resp.EventStream.Err(); err != nil {
fmt.Fprintf(os.Stderr, "reading from event stream failed, %v\n", err)
}
}
func BenchmarkSelectObjectContentEventStream(b *testing.B) {
_, eventMsgs := mockSelectObjectContentEventStreamsOutputEvents()
var buf bytes.Buffer
encoder := eventstream.NewEncoder(&buf)
for _, msg := range eventMsgs {
if err := encoder.Encode(msg); err != nil {
b.Fatalf("failed to encode message, %v", err)
}
}
stream := &loopReader{source: bytes.NewReader(buf.Bytes())}
sess := unit.Session
svc := New(sess, &aws.Config{
Endpoint: aws.String("https://example.com"),
DisableParamValidation: aws.Bool(true),
})
svc.Handlers.Send.Swap(corehandlers.SendHandler.Name,
request.NamedHandler{Name: "mockSend",
Fn: func(r *request.Request) {
r.HTTPResponse = &http.Response{
Status: "200 OK",
StatusCode: 200,
Header: http.Header{},
Body: ioutil.NopCloser(stream),
}
},
},
)
params := &SelectObjectContentInput{}
resp, err := svc.SelectObjectContent(params)
if err != nil {
b.Fatalf("failed to create request, %v", err)
}
defer resp.EventStream.Close()
b.ResetTimer()
for i := 0; i < b.N; i++ {
if err = resp.EventStream.Err(); err != nil {
b.Fatalf("expect no error, got %v", err)
}
event := <-resp.EventStream.Events()
if event == nil {
b.Fatalf("expect event, got nil, %v, %d", resp.EventStream.Err(), i)
}
}
}
type loopReader struct {
source *bytes.Reader
}
func (c *loopReader) Read(p []byte) (int, error) {
if c.source.Len() == 0 {
c.source.Seek(0, 0)
}
return c.source.Read(p)
}

View file

@ -1,7 +1,10 @@
package s3_test
import (
"encoding/json"
"net/url"
"os"
"path/filepath"
"strings"
"testing"
@ -125,3 +128,52 @@ func TestHostStyleBucketGetBucketLocation(t *testing.T) {
t.Errorf("expect %s to be in %s", e, a)
}
}
func TestVirtualHostStyleSuite(t *testing.T) {
f, err := os.Open(filepath.Join("testdata", "virtual_host.json"))
if err != nil {
t.Fatalf("expect no error, %v", err)
}
cases := []struct {
Bucket string
Region string
UseDualStack bool
UseS3Accelerate bool
ConfiguredAddressingStyle string
ExpectedURI string
}{}
decoder := json.NewDecoder(f)
if err := decoder.Decode(&cases); err != nil {
t.Fatalf("expect no error, %v", err)
}
const testPathStyle = "path"
for i, c := range cases {
svc := s3.New(unit.Session, &aws.Config{
Region: &c.Region,
UseDualStack: &c.UseDualStack,
S3UseAccelerate: &c.UseS3Accelerate,
S3ForcePathStyle: aws.Bool(c.ConfiguredAddressingStyle == testPathStyle),
})
req, _ := svc.HeadBucketRequest(&s3.HeadBucketInput{
Bucket: &c.Bucket,
})
req.Build()
if req.Error != nil {
t.Fatalf("expect no error, got %v", req.Error)
}
// Trim trailing '/' that are added by the SDK but not in the tests.
actualURI := strings.TrimRightFunc(
req.HTTPRequest.URL.String(),
func(r rune) bool { return r == '/' },
)
if e, a := c.ExpectedURI, actualURI; e != a {
t.Errorf("%d, expect\n%s\nurl to be\n%s", i, e, a)
}
}
}

View file

@ -375,6 +375,10 @@ type S3API interface {
RestoreObjectWithContext(aws.Context, *s3.RestoreObjectInput, ...request.Option) (*s3.RestoreObjectOutput, error)
RestoreObjectRequest(*s3.RestoreObjectInput) (*request.Request, *s3.RestoreObjectOutput)
SelectObjectContent(*s3.SelectObjectContentInput) (*s3.SelectObjectContentOutput, error)
SelectObjectContentWithContext(aws.Context, *s3.SelectObjectContentInput, ...request.Option) (*s3.SelectObjectContentOutput, error)
SelectObjectContentRequest(*s3.SelectObjectContentInput) (*request.Request, *s3.SelectObjectContentOutput)
UploadPart(*s3.UploadPartInput) (*s3.UploadPartOutput, error)
UploadPartWithContext(aws.Context, *s3.UploadPartInput, ...request.Option) (*s3.UploadPartOutput, error)
UploadPartRequest(*s3.UploadPartInput) (*request.Request, *s3.UploadPartOutput)

View file

@ -31,6 +31,7 @@ var initRequest func(*request.Request)
const (
ServiceName = "s3" // Service endpoint prefix API calls made to.
EndpointsID = ServiceName // Service ID for Regions and Endpoints metadata.
ServiceID = "S3" // ServiceID is a unique identifer of a specific service
)
// New creates a new instance of the S3 client with a session.
@ -55,6 +56,7 @@ func newClient(cfg aws.Config, handlers request.Handlers, endpoint, signingRegio
cfg,
metadata.ClientInfo{
ServiceName: ServiceName,
ServiceID: ServiceID,
SigningName: signingName,
SigningRegion: signingRegion,
Endpoint: endpoint,
@ -71,6 +73,8 @@ func newClient(cfg aws.Config, handlers request.Handlers, endpoint, signingRegio
svc.Handlers.UnmarshalMeta.PushBackNamed(restxml.UnmarshalMetaHandler)
svc.Handlers.UnmarshalError.PushBackNamed(restxml.UnmarshalErrorHandler)
svc.Handlers.UnmarshalStream.PushBackNamed(restxml.UnmarshalHandler)
// Run custom client initialization if present
if initClient != nil {
initClient(svc.Client)

View file

@ -0,0 +1,178 @@
[
{
"Bucket": "bucket-name",
"ConfiguredAddressingStyle": "default",
"ExpectedUri": "https://bucket-name.s3.amazonaws.com",
"Region": "us-east-1",
"UseDualstack": false,
"UseS3Accelerate": false
},
{
"Bucket": "bucket-name",
"ConfiguredAddressingStyle": "default",
"ExpectedUri": "https://bucket-name.s3.us-west-1.amazonaws.com",
"Region": "us-west-1",
"UseDualstack": false,
"UseS3Accelerate": false
},
{
"Bucket": "bucket-with-number-1",
"ConfiguredAddressingStyle": "default",
"ExpectedUri": "https://bucket-with-number-1.s3.us-west-1.amazonaws.com",
"Region": "us-west-1",
"UseDualstack": false,
"UseS3Accelerate": false
},
{
"Bucket": "bucket-name",
"ConfiguredAddressingStyle": "default",
"ExpectedUri": "https://bucket-name.s3.cn-north-1.amazonaws.com.cn",
"Region": "cn-north-1",
"UseDualstack": false,
"UseS3Accelerate": false
},
{
"Bucket": "BucketName",
"ConfiguredAddressingStyle": "default",
"ExpectedUri": "https://s3.amazonaws.com/BucketName",
"Region": "us-east-1",
"UseDualstack": false,
"UseS3Accelerate": false
},
{
"Bucket": "BucketName",
"ConfiguredAddressingStyle": "default",
"ExpectedUri": "https://s3.amazonaws.com/BucketName",
"Region": "us-east-1",
"UseDualstack": false,
"UseS3Accelerate": false
},
{
"Bucket": "bucket_name",
"ConfiguredAddressingStyle": "default",
"ExpectedUri": "https://s3.us-west-1.amazonaws.com/bucket_name",
"Region": "us-west-1",
"UseDualstack": false,
"UseS3Accelerate": false
},
{
"Bucket": "bucket.name",
"ConfiguredAddressingStyle": "default",
"ExpectedUri": "https://s3.us-west-1.amazonaws.com/bucket.name",
"Region": "us-west-1",
"UseDualstack": false,
"UseS3Accelerate": false
},
{
"Bucket": "-bucket-name",
"ConfiguredAddressingStyle": "default",
"ExpectedUri": "https://s3.us-west-1.amazonaws.com/-bucket-name",
"Region": "us-west-1",
"UseDualstack": false,
"UseS3Accelerate": false
},
{
"Bucket": "bucket-name-",
"ConfiguredAddressingStyle": "default",
"ExpectedUri": "https://s3.us-west-1.amazonaws.com/bucket-name-",
"Region": "us-west-1",
"UseDualstack": false,
"UseS3Accelerate": false
},
{
"Bucket": "aa",
"ConfiguredAddressingStyle": "default",
"ExpectedUri": "https://s3.us-west-1.amazonaws.com/aa",
"Region": "us-west-1",
"UseDualstack": false,
"UseS3Accelerate": false
},
{
"Bucket": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
"ConfiguredAddressingStyle": "default",
"ExpectedUri": "https://s3.us-west-1.amazonaws.com/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
"Region": "us-west-1",
"UseDualstack": false,
"UseS3Accelerate": false
},
{
"Bucket": "bucket-name",
"ConfiguredAddressingStyle": "default",
"ExpectedUri": "https://bucket-name.s3-accelerate.amazonaws.com",
"Region": "us-east-1",
"UseDualstack": false,
"UseS3Accelerate": true
},
{
"Bucket": "bucket-name",
"ConfiguredAddressingStyle": "default",
"ExpectedUri": "https://bucket-name.s3-accelerate.amazonaws.com",
"Region": "us-west-1",
"UseDualstack": false,
"UseS3Accelerate": true
},
{
"Bucket": "bucket-name",
"ConfiguredAddressingStyle": "default",
"ExpectedUri": "https://bucket-name.s3.dualstack.us-east-1.amazonaws.com",
"Region": "us-east-1",
"UseDualstack": true,
"UseS3Accelerate": false
},
{
"Bucket": "bucket-name",
"ConfiguredAddressingStyle": "default",
"ExpectedUri": "https://bucket-name.s3.dualstack.us-west-2.amazonaws.com",
"Region": "us-west-2",
"UseDualstack": true,
"UseS3Accelerate": false
},
{
"Bucket": "bucket.name",
"ConfiguredAddressingStyle": "default",
"ExpectedUri": "https://s3.dualstack.us-west-2.amazonaws.com/bucket.name",
"Region": "us-west-2",
"UseDualstack": true,
"UseS3Accelerate": false
},
{
"Bucket": "bucket-name",
"ConfiguredAddressingStyle": "default",
"ExpectedUri": "https://bucket-name.s3-accelerate.dualstack.amazonaws.com",
"Region": "us-east-1",
"UseDualstack": true,
"UseS3Accelerate": true
},
{
"Bucket": "bucket-name",
"ConfiguredAddressingStyle": "path",
"ExpectedUri": "https://s3.amazonaws.com/bucket-name",
"Region": "us-east-1",
"UseDualstack": false,
"UseS3Accelerate": false
},
{
"Bucket": "bucket-name",
"ConfiguredAddressingStyle": "path",
"ExpectedUri": "https://bucket-name.s3-accelerate.amazonaws.com",
"Region": "us-east-1",
"UseDualstack": false,
"UseS3Accelerate": true
},
{
"Bucket": "bucket-name",
"ConfiguredAddressingStyle": "path",
"ExpectedUri": "https://s3.dualstack.us-east-1.amazonaws.com/bucket-name",
"Region": "us-east-1",
"UseDualstack": true,
"UseS3Accelerate": false
},
{
"Bucket": "bucket-name",
"ConfiguredAddressingStyle": "path",
"ExpectedUri": "https://bucket-name.s3-accelerate.dualstack.amazonaws.com",
"Region": "us-east-1",
"UseDualstack": true,
"UseS3Accelerate": true
}
]