TrueCloudLab/rclone
16
0
Fork 4
Code Issues 1 Pull requests Projects Releases 4 Packages Wiki Activity Actions

vendor: update all dependencies

Browse source
This commit is contained in:
Nick Craig-Wood 2018-03-19 15:51:38 +00:00
parent 940df88eb2
commit d64789528d
4309 changed files with 1327278 additions and 1001118 deletions
Download patch file Download diff file Expand all files Collapse all files

7
vendor/github.com/yunify/qingstor-sdk-go/CHANGELOG.md generated vendored
View file

@ -1,6 +1,13 @@
# Change Log
All notable changes to QingStor SDK for Go will be documented in this file.
## [v2.2.10] - 2018-3-14
### Changed
- Close body for every API except GetObject and ImageProcess
- Add correct i/o timeout behavior for http client
## [v2.2.9] - 2017-11-25
### Changed

110
vendor/github.com/yunify/qingstor-sdk-go/config/config.go generated vendored
View file

@ -27,6 +27,7 @@ import (
"gopkg.in/yaml.v2"
"github.com/yunify/qingstor-sdk-go/logger"
"github.com/yunify/qingstor-sdk-go/utils"
)
// A Config stores a configuration of this sdk.
@ -43,9 +44,55 @@ type Config struct {
LogLevel string `yaml:"log_level"`
HTTPSettings HTTPClientSettings
Connection *http.Client
}
// HTTPClientSettings is the http client settings.
type HTTPClientSettings struct {
// ConnectTimeout affects making new socket connection
ConnectTimeout time.Duration `yaml:"connect_timeout"`
// ReadTimeout affect each call to HTTPResponse.Body.Read()
ReadTimeout time.Duration `yaml:"read_timeout"`
// WriteTimeout affect each write in io.Copy while sending HTTPRequest
WriteTimeout time.Duration `yaml:"write_timeout" `
// TLSHandshakeTimeout affects https hand shake timeout
TLSHandshakeTimeout time.Duration `yaml:"tls_timeout"`
// IdleConnTimeout affects the time limit to re-use http connections
IdleConnTimeout time.Duration `yaml:"idle_timeout"`
TCPKeepAlive time.Duration `yaml:"tcp_keepalive_time"`
DualStack bool `yaml:"dual_stack"`
// MaxIdleConns affects the idle connections kept for re-use
MaxIdleConns int `yaml:"max_idle_conns"`
MaxIdleConnsPerHost int `yaml:"max_idle_conns_per_host"`
ExpectContinueTimeout time.Duration `yaml:"expect_continue_timeout"`
}
// DefaultHTTPClientSettings is the default http client settings.
var DefaultHTTPClientSettings = HTTPClientSettings{
ConnectTimeout: time.Second * 30,
ReadTimeout: time.Second * 30,
WriteTimeout: time.Second * 30,
TLSHandshakeTimeout: time.Second * 10,
IdleConnTimeout: time.Second * 20,
TCPKeepAlive: 0,
DualStack: false,
MaxIdleConns: 100,
MaxIdleConnsPerHost: 10,
ExpectContinueTimeout: time.Second * 2,
}
// New create a Config with given AccessKeyID and SecretAccessKey.
func New(accessKeyID, secretAccessKey string) (c *Config, err error) {
c, err = NewDefault()
@ -53,14 +100,9 @@ func New(accessKeyID, secretAccessKey string) (c *Config, err error) {
c = nil
return
}
c.AccessKeyID = accessKeyID
c.SecretAccessKey = secretAccessKey
c.Connection = &http.Client{
Timeout: time.Minute,
}
return
}
@ -72,10 +114,6 @@ func NewDefault() (c *Config, err error) {
c = nil
return
}
c.Connection = &http.Client{
Timeout: time.Minute,
}
return
}
@ -124,6 +162,9 @@ func (c *Config) Check() (err error) {
// LoadDefaultConfig loads the default configuration for Config.
// It returns error if yaml decode failed.
func (c *Config) LoadDefaultConfig() (err error) {
c.HTTPSettings = DefaultHTTPClientSettings
err = yaml.Unmarshal([]byte(DefaultConfigFileContent), c)
if err != nil {
logger.Errorf(nil, "Config parse error, %v.", err)
@ -131,6 +172,8 @@ func (c *Config) LoadDefaultConfig() (err error) {
}
logger.SetLevel(c.LogLevel)
c.InitHTTPClient()
return
}
@ -179,5 +222,54 @@ func (c *Config) LoadConfigFromContent(content []byte) (err error) {
}
logger.SetLevel(c.LogLevel)
c.InitHTTPClient()
return
}
// InitHTTPClient : After modifying Config.HTTPSettings, you should always call this to initialize the HTTP Client.
func (c *Config) InitHTTPClient() {
var emptySettings HTTPClientSettings
if c.HTTPSettings == emptySettings { // User forgot to initialize the settings
c.HTTPSettings = DefaultHTTPClientSettings
} else {
if c.HTTPSettings.ConnectTimeout == 0 {
c.HTTPSettings.ConnectTimeout = DefaultHTTPClientSettings.ConnectTimeout
}
// If ReadTimeout and WriteTimeout is zero, means no read/write timeout
if c.HTTPSettings.TLSHandshakeTimeout == 0 {
c.HTTPSettings.TLSHandshakeTimeout = DefaultHTTPClientSettings.TLSHandshakeTimeout
}
if c.HTTPSettings.ExpectContinueTimeout == 0 {
c.HTTPSettings.ExpectContinueTimeout = DefaultHTTPClientSettings.ExpectContinueTimeout
}
}
dialer := utils.NewDialer(
c.HTTPSettings.ConnectTimeout,
c.HTTPSettings.ReadTimeout,
c.HTTPSettings.WriteTimeout,
)
dialer.KeepAlive = c.HTTPSettings.TCPKeepAlive
// XXX: DualStack enables RFC 6555-compliant "Happy Eyeballs" dialing
// when the network is "tcp" and the destination is a host name
// with both IPv4 and IPv6 addresses. This allows a client to
// tolerate networks where one address family is silently broken
dialer.DualStack = c.HTTPSettings.DualStack
c.Connection = &http.Client{
// We do not use the timeout in http client,
// because this timeout is for the whole http body read/write,
// it's unsuitable for various length of files and network condition.
// We provide a wraper in utils/conn.go of net.Dialer to make io timeout to the http connection
// for individual buffer I/O operation,
Timeout: 0,
Transport: &http.Transport{
DialContext: dialer.DialContext,
MaxIdleConns: c.HTTPSettings.MaxIdleConns,
MaxIdleConnsPerHost: c.HTTPSettings.MaxIdleConnsPerHost,
IdleConnTimeout: c.HTTPSettings.IdleConnTimeout,
TLSHandshakeTimeout: c.HTTPSettings.TLSHandshakeTimeout,
ExpectContinueTimeout: c.HTTPSettings.ExpectContinueTimeout,
},
}
}

13
vendor/github.com/yunify/qingstor-sdk-go/docs/configuration.md generated vendored
View file

@ -67,3 +67,16 @@ moreConfiguration.Protocol = "http"
moreConfiguration.Host = "api.private.com"
moreConfiguration.Port = 80
```
Change http timeout
``` go
customConfiguration, _ := config.NewDefault().LoadUserConfig()
// For the default value refers to DefaultHTTPClientSettings in config package
// ReadTimeout affect each call to HTTPResponse.Body.Read()
customConfiguration.HTTPSettings.ReadTimeout = 2 * time.Minute
// WriteTimeout affect each write in io.Copy while sending HTTPRequest
customConfiguration.HTTPSettings.WriteTimeout = 2 * time.Minute
// Re-initialize the client to take effect
customConfiguration.InitHTTPClient()
```

1
vendor/github.com/yunify/qingstor-sdk-go/docs/image_process_usage.md generated vendored
View file

@ -253,6 +253,7 @@ func main() {
imageProcessOutput, err = bucket.ImageProcess("imageName", &qs.ImageProcessInput{
Action: &operation})
checkErr(err)
defer imageProcessOutput.Close() // Don't forget to close the output otherwise will be leaking http connections
testOutput(imageProcessOutput)
}

86
vendor/github.com/yunify/qingstor-sdk-go/docs/qingstor_service_usage.md generated vendored
View file

@ -10,7 +10,9 @@ You can use a specified version of a service by import a service package with a
``` go
import (
// Import the latest version API
"github.com/yunify/qingstor-sdk-go/config"
qs "github.com/yunify/qingstor-sdk-go/service"
qsErrors "github.com/yunify/qingstor-sdk-go/request/errors"
)
```
@ -19,13 +21,15 @@ import (
Initialize the QingStor service with a configuration
``` go
qsService, _ := qs.Init(configuration)
userConfig, err := config.NewDefault().LoadUserConfig()
if err != nil { panic(err) }
qsService, _ := qs.Init(userConfig)
```
List buckets
``` go
qsOutput, _ := qsService.ListBuckets(nil)
qsOutput, err := qsService.ListBuckets(nil)
// Print the HTTP status code.
// Example: 200
@ -43,13 +47,13 @@ fmt.Println(qs.String(qsOutput.Buckets[0].Name))
Initialize a QingStor bucket
``` go
bucket, _ := qsService.Bucket("test-bucket", "pek3a")
bucket, err := qsService.Bucket("test-bucket", "pek3a")
```
List objects in the bucket
``` go
bOutput, _ := bucket.ListObjects(nil)
bOutput, err := bucket.ListObjects(nil)
// Print the HTTP status code.
// Example: 200
@ -63,7 +67,7 @@ fmt.Println(len(bOutput.Keys))
Set ACL of the bucket
``` go
bACLOutput, _ := bucket.PutACL(&qs.PutBucketACLInput{
bACLOutput, err := bucket.PutACL(&qs.PutBucketACLInput{
ACL: []*service.ACLType{{
Grantee: &service.GranteeType{
Type: qs.String("user"),
@ -82,7 +86,7 @@ Put object
``` go
// Open file
file, _ := os.Open("/tmp/Screenshot.jpg")
file, err := os.Open("/tmp/Screenshot.jpg")
defer file.Close()
// Calculate MD5
@ -92,7 +96,7 @@ hashInBytes := hash.Sum(nil)[:16]
md5String := hex.EncodeToString(hashInBytes)
// Put object
oOutput, _ := bucket.PutObject(
putOutput, err := bucket.PutObject(
"Screenshot.jpg",
&service.PutObjectInput{
ContentLength: qs.Int(102475), // Obtain automatically if empty
@ -101,26 +105,58 @@ oOutput, _ := bucket.PutObject(
Body: file,
},
)
// Print the HTTP status code.
// Example: 201
fmt.Println(qs.IntValue(oOutput.StatusCode))
if err != nil {
// Example: QingStor Error: StatusCode 403, Code "permission_denied"...
fmt.Println(err)
} else {
// Print the HTTP status code.
// Example: 201
fmt.Println(qs.IntValue(putOutput.StatusCode))
}
```
Get object
``` go
getOutput, err := bucket.GetObject(
"Screenshot.jpg",
&GetObjectInput{},
)
if err != nil {
// Example: QingStor Error: StatusCode 404, Code "object_not_exists"...
fmt.Println(err)
if qsErr, ok := err.(*qsErrors.QingStorError); ok {
println(qsErr.StatusCode, qsErr.Code)
}
} else {
defer getOutput.Close() // Don't forget to close, otherwise will be leaking connections
f, err := os.OpenFile("download_screenshot.jpg", os.O_CREATE|os.O_WRONLY, 0600)
if err != nil {
panic(err)
}
defer f.Close()
if _, err = io.Copy(f, getOutput.Body); err != nil {
panic(err)
}
}
```
Delete object
``` go
oOutput, _ := bucket.DeleteObject("Screenshot.jpg")
delOutput, err := bucket.DeleteObject("Screenshot.jpg")
// Print the HTTP status code.
// Example: 204
fmt.Println(qs.IntValue(oOutput.StatusCode))
fmt.Println(qs.IntValue(delOutput.StatusCode))
```
Initialize Multipart Upload
``` go
aOutput, _ := bucket.InitiateMultipartUpload(
initOutput, err := bucket.InitiateMultipartUpload(
"QingCloudInsight.mov",
&service.InitiateMultipartUploadInput{
ContentType: qs.String("video/quicktime"),
@ -129,17 +165,17 @@ aOutput, _ := bucket.InitiateMultipartUpload(
// Print the HTTP status code.
// Example: 200
fmt.Println(qs.IntValue(aOutput.StatusCode))
fmt.Println(qs.IntValue(initOutput.StatusCode))
// Print the upload ID.
// Example: "9d37dd6ccee643075ca4e597ad65655c"
fmt.Println(qs.StringValue(aOutput.UploadID))
fmt.Println(qs.StringValue(initOutput.UploadID))
```
Upload Multipart
``` go
aOutput, _ := bucket.UploadMultipart(
uploadOutput, err := bucket.UploadMultipart(
"QingCloudInsight.mov",
&service.UploadMultipartInput{
UploadID: qs.String("xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"),
@ -151,9 +187,9 @@ aOutput, _ := bucket.UploadMultipart(
// Print the HTTP status code.
// Example: 201
fmt.Println(qs.IntValue(aOutput.StatusCode))
fmt.Println(qs.IntValue(uploadOutput.StatusCode))
aOutput, _ = bucket.UploadMultipart(
uploadOutput, err = bucket.UploadMultipart(
"QingCloudInsight.mov",
&service.UploadMultipartInput{
UploadID: qs.String("xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"),
@ -165,9 +201,9 @@ aOutput, _ = bucket.UploadMultipart(
// Print the HTTP status code.
// Example: 201
fmt.Println(qs.IntValue(aOutput.StatusCode))
fmt.Println(qs.IntValue(uploadOutput.StatusCode))
aOutput, _ = bucket.UploadMultipart(
uploadOutput, err = bucket.UploadMultipart(
"QingCloudInsight.mov"
&service.UploadMultipartInput{
UploadID: qs.String("xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"),
@ -179,13 +215,13 @@ aOutput, _ = bucket.UploadMultipart(
// Print the HTTP status code.
// Example: 201
fmt.Println(qs.IntValue(aOutput.StatusCode))
fmt.Println(qs.IntValue(uploadOutput.StatusCode))
```
Complete Multipart Upload
``` go
aOutput, _ := bucket.CompleteMultipartUpload(
completeOutput, err := bucket.CompleteMultipartUpload(
"QingCloudInsight.mov",
&service.CompleteMultipartUploadInput{
UploadID: qs.String("xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"),
@ -201,13 +237,13 @@ aOutput, _ := bucket.CompleteMultipartUpload(
// Print the HTTP status code.
// Example: 200
fmt.Println(qs.IntValue(aOutput.StatusCode))
fmt.Println(qs.IntValue(completeOutput.StatusCode))
```
Abort Multipart Upload
``` go
aOutput, err := bucket.AbortMultipartUpload(
abrtOutput, err := bucket.AbortMultipartUpload(
"QingCloudInsight.mov"
&service.AbortMultipartUploadInput{
UploadID: qs.String("xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"),

3
vendor/github.com/yunify/qingstor-sdk-go/request/request.go generated vendored
View file

@ -111,7 +111,6 @@ func (r *Request) Do() error {
if err != nil {
return err
}
return nil
}
@ -210,7 +209,7 @@ func (r *Request) send() error {
var err error
if r.Operation.Config.Connection == nil {
return errors.New("connection not initialized")
r.Operation.Config.InitHTTPClient()
}
retries := r.Operation.Config.ConnectionRetries + 1

10
vendor/github.com/yunify/qingstor-sdk-go/request/unpacker/qingstor.go generated vendored
View file

@ -44,6 +44,16 @@ func (qu *QingStorUnpacker) UnpackHTTPRequest(o *data.Operation, r *http.Respons
return err
}
// Close body for every API except GetObject and ImageProcess.
if o.APIName != "GET Object" && o.APIName != "Image Process" && r.Body != nil {
err = r.Body.Close()
if err != nil {
return err
}
r.Body = nil
}
return nil
}

16
vendor/github.com/yunify/qingstor-sdk-go/service/object.go generated vendored
View file

@ -380,6 +380,14 @@ type GetObjectOutput struct {
XQSEncryptionCustomerAlgorithm *string `json:"X-QS-Encryption-Customer-Algorithm,omitempty" name:"X-QS-Encryption-Customer-Algorithm" location:"headers"`
}
// Close will close the underlay body.
func (o *GetObjectOutput) Close() (err error) {
if o.Body != nil {
return o.Body.Close()
}
return
}
// HeadObject does Check whether the object exists and available.
// Documentation URL: https://docs.qingcloud.com/qingstor/api/object/head.html
func (s *Bucket) HeadObject(objectKey string, input *HeadObjectInput) (*HeadObjectOutput, error) {
@ -571,6 +579,14 @@ type ImageProcessOutput struct {
ContentLength *int64 `json:"Content-Length,omitempty" name:"Content-Length" location:"headers"`
}
// Close will close the underlay body.
func (o *ImageProcessOutput) Close() (err error) {
if o.Body != nil {
return o.Body.Close()
}
return
}
// InitiateMultipartUpload does Initial multipart upload on the object.
// Documentation URL: https://docs.qingcloud.com/qingstor/api/object/initiate_multipart_upload.html
func (s *Bucket) InitiateMultipartUpload(objectKey string, input *InitiateMultipartUploadInput) (*InitiateMultipartUploadOutput, error) {

11
vendor/github.com/yunify/qingstor-sdk-go/template/shared.tmpl generated vendored
View file

@ -293,6 +293,17 @@
{{end}}
{{end}}
}
{{range $keyStatus, $valueStatus := $operation.Responses -}}
{{if eq $valueStatus.Body.Type "binary"}}
// Close will close the underlay body.
func (o *{{$opID}}Output) Close() (err error) {
if o.Body != nil {
return o.Body.Close()
}
return
}
{{end}}
{{end}}
{{end}}
{{define "SubServiceInitParams"}}

136
vendor/github.com/yunify/qingstor-sdk-go/utils/conn.go generated vendored Normal file
View file

@ -0,0 +1,136 @@
package utils
import (
"context"
"net"
"sync"
"time"
)
var connPool sync.Pool
type netConn net.Conn
type netDialer net.Dialer
// Dialer is wrapped dialer provided by qingstor go sdk.
//
// We provide this dialer wrapper ReadTimeout & WriteTimeout attributes into connection object.
// This timeout is for individual buffer I/O operation like other language (python, perl... etc),
// so don't bother with SetDeadline or stupid nethttp.Client timeout.
type Dialer struct {
*net.Dialer
ReadTimeout time.Duration
WriteTimeout time.Duration
}
// NewDialer will create a new dialer.
func NewDialer(connTimeout, readTimeout, writeTimeout time.Duration) *Dialer {
d := &net.Dialer{
DualStack: false,
Timeout: connTimeout,
}
return &Dialer{d, readTimeout, writeTimeout}
}
// Dial connects to the address on the named network.
func (d *Dialer) Dial(network, addr string) (net.Conn, error) {
c, err := d.Dialer.Dial(network, addr)
if err != nil {
return nil, err
}
conn := NewConn(c)
conn.readTimeout = d.ReadTimeout
conn.writeTimeout = d.WriteTimeout
return conn, nil
}
// DialContext connects to the address on the named network using
// the provided context.
func (d *Dialer) DialContext(ctx context.Context, network, addr string) (net.Conn, error) {
c, err := d.Dialer.DialContext(ctx, network, addr)
if err != nil {
return nil, err
}
conn := NewConn(c)
conn.readTimeout = d.ReadTimeout
conn.writeTimeout = d.WriteTimeout
return conn, nil
}
// Conn is a generic stream-oriented network connection.
type Conn struct {
netConn
readTimeout time.Duration
writeTimeout time.Duration
timeoutFunc func() bool
}
// NewConn will create a new conn.
func NewConn(c netConn) *Conn {
conn, ok := c.(*Conn)
if ok {
return conn
}
conn, ok = connPool.Get().(*Conn)
if !ok {
conn = new(Conn)
}
conn.netConn = c
return conn
}
// SetReadTimeout will set the conn's read timeout.
func (c *Conn) SetReadTimeout(d time.Duration) {
c.readTimeout = d
}
// SetWriteTimeout will set the conn's write timeout.
func (c *Conn) SetWriteTimeout(d time.Duration) {
c.writeTimeout = d
}
// Read will read from the conn.
func (c Conn) Read(buf []byte) (n int, err error) {
if c.readTimeout > 0 {
c.SetDeadline(time.Now().Add(c.readTimeout))
}
n, err = c.netConn.Read(buf)
if c.readTimeout > 0 {
c.SetDeadline(time.Time{}) // clear timeout
}
return
}
// Write will write into the conn.
func (c Conn) Write(buf []byte) (n int, err error) {
if c.writeTimeout > 0 {
c.SetDeadline(time.Now().Add(c.writeTimeout))
}
n, err = c.netConn.Write(buf)
if c.writeTimeout > 0 {
c.SetDeadline(time.Time{})
}
return
}
// Close will close the conn.
func (c Conn) Close() (err error) {
if c.netConn == nil {
return nil
}
err = c.netConn.Close()
connPool.Put(c)
c.netConn = nil
c.readTimeout = 0
c.writeTimeout = 0
return
}
// IsTimeoutError will check whether the err is a timeout error.
func IsTimeoutError(err error) bool {
e, ok := err.(net.Error)
if ok {
return e.Timeout()
}
return false
}

2
vendor/github.com/yunify/qingstor-sdk-go/version.go generated vendored
View file

@ -20,4 +20,4 @@
package sdk
// Version number.
const Version = "2.2.9"
const Version = "2.2.10"