potyarkin/rclone
1
0
Fork 0
forked from TrueCloudLab/rclone
Code Pull requests Activity

vendor: update all dependencies

Browse source
This commit is contained in:
Nick Craig-Wood 2017-07-23 08:51:42 +01:00
parent 0b6fba34a3
commit eb87cf6f12
2008 changed files with 352633 additions and 1004750 deletions
Download patch file Download diff file Expand all files Collapse all files

165
vendor/github.com/ncw/swift/swift.go generated vendored
View file

@ -11,9 +11,11 @@ import (
"fmt"
"hash"
"io"
"io/ioutil"
"mime"
"net/http"
"net/url"
"os"
"path"
"strconv"
"strings"
@ -123,6 +125,135 @@ type Connection struct {
swiftInfo SwiftInfo
}
// setFromEnv reads the value that param points to (it must be a
// pointer), if it isn't the zero value then it reads the environment
// variable name passed in, parses it according to the type and writes
// it to the pointer.
func setFromEnv(param interface{}, name string) (err error) {
val := os.Getenv(name)
if val == "" {
return
}
switch result := param.(type) {
case *string:
if *result == "" {
*result = val
}
case *int:
if *result == 0 {
*result, err = strconv.Atoi(val)
}
case *bool:
if *result == false {
*result, err = strconv.ParseBool(val)
}
case *time.Duration:
if *result == 0 {
*result, err = time.ParseDuration(val)
}
case *EndpointType:
if *result == EndpointType("") {
*result = EndpointType(val)
}
default:
return newErrorf(0, "can't set var of type %T", param)
}
return err
}
// ApplyEnvironment reads environment variables and applies them to
// the Connection structure. It won't overwrite any parameters which
// are already set in the Connection struct.
//
// To make a new Connection object entirely from the environment you
// would do:
//
// c := new(Connection)
// err := c.ApplyEnvironment()
// if err != nil { log.Fatal(err) }
//
// The naming of these variables follows the official Openstack naming
// scheme so it should be compatible with OpenStack rc files.
//
// For v1 authentication (obsolete)
// ST_AUTH - Auth URL
// ST_USER - UserName for api
// ST_KEY - Key for api access
//
// For v2 authentication
// OS_AUTH_URL - Auth URL
// OS_USERNAME - UserName for api
// OS_PASSWORD - Key for api access
// OS_TENANT_NAME - Name of the tenant
// OS_TENANT_ID - Id of the tenant
// OS_REGION_NAME - Region to use - default is use first region
//
// For v3 authentication
// OS_AUTH_URL - Auth URL
// OS_USERNAME - UserName for api
// OS_PASSWORD - Key for api access
// OS_USER_DOMAIN_NAME - User's domain name
// OS_USER_DOMAIN_ID - User's domain Id
// OS_PROJECT_NAME - Name of the project
// OS_PROJECT_DOMAIN_NAME - Name of the tenant's domain, only needed if it differs from the user domain
// OS_PROJECT_DOMAIN_ID - Id of the tenant's domain, only needed if it differs the from user domain
// OS_TRUST_ID - If of the trust
// OS_REGION_NAME - Region to use - default is use first region
//
// Other
// OS_ENDPOINT_TYPE - Endpoint type public, internal or admin
// ST_AUTH_VERSION - Choose auth version - 1, 2 or 3 or leave at 0 for autodetect
//
// For manual authentication
// OS_STORAGE_URL - storage URL from alternate authentication
// OS_AUTH_TOKEN - Auth Token from alternate authentication
//
// Library specific
// GOSWIFT_RETRIES - Retries on error (default is 3)
// GOSWIFT_USER_AGENT - HTTP User agent (default goswift/1.0)
// GOSWIFT_CONNECT_TIMEOUT - Connect channel timeout with unit, eg "10s", "100ms" (default "10s")
// GOSWIFT_TIMEOUT - Data channel timeout with unit, eg "10s", "100ms" (default "60s")
// GOSWIFT_INTERNAL - Set this to "true" to use the the internal network (obsolete - use OS_ENDPOINT_TYPE)
func (c *Connection) ApplyEnvironment() (err error) {
for _, item := range []struct {
result interface{}
name string
}{
// Environment variables - keep in same order as Connection
{&c.Domain, "OS_USER_DOMAIN_NAME"},
{&c.DomainId, "OS_USER_DOMAIN_ID"},
{&c.UserName, "OS_USERNAME"},
{&c.ApiKey, "OS_PASSWORD"},
{&c.AuthUrl, "OS_AUTH_URL"},
{&c.Retries, "GOSWIFT_RETRIES"},
{&c.UserAgent, "GOSWIFT_USER_AGENT"},
{&c.ConnectTimeout, "GOSWIFT_CONNECT_TIMEOUT"},
{&c.Timeout, "GOSWIFT_TIMEOUT"},
{&c.Region, "OS_REGION_NAME"},
{&c.AuthVersion, "ST_AUTH_VERSION"},
{&c.Internal, "GOSWIFT_INTERNAL"},
{&c.Tenant, "OS_TENANT_NAME"}, //v2
{&c.Tenant, "OS_PROJECT_NAME"}, // v3
{&c.TenantId, "OS_TENANT_ID"},
{&c.EndpointType, "OS_ENDPOINT_TYPE"},
{&c.TenantDomain, "OS_PROJECT_DOMAIN_NAME"},
{&c.TenantDomainId, "OS_PROJECT_DOMAIN_ID"},
{&c.TrustId, "OS_TRUST_ID"},
{&c.StorageUrl, "OS_STORAGE_URL"},
{&c.AuthToken, "OS_AUTH_TOKEN"},
// v1 auth alternatives
{&c.ApiKey, "ST_KEY"},
{&c.UserName, "ST_USER"},
{&c.AuthUrl, "ST_AUTH"},
} {
err = setFromEnv(item.result, item.name)
if err != nil {
return newErrorf(0, "failed to read env var %q: %v", item.name, err)
}
}
return nil
}
// Error - all errors generated by this package are of this type. Other error
// may be passed on from library functions though.
type Error struct {
@ -197,15 +328,32 @@ func checkClose(c io.Closer, err *error) {
}
}
// drainAndClose discards all data from rd and closes it.
// If an error occurs during Read, it is discarded.
func drainAndClose(rd io.ReadCloser, err *error) {
if rd == nil {
return
}
_, _ = io.Copy(ioutil.Discard, rd)
cerr := rd.Close()
if err != nil && *err == nil {
*err = cerr
}
}
// parseHeaders checks a response for errors and translates into
// standard errors if necessary.
// standard errors if necessary. If an error is returned, resp.Body
// has been drained and closed.
func (c *Connection) parseHeaders(resp *http.Response, errorMap errorMap) error {
if errorMap != nil {
if err, ok := errorMap[resp.StatusCode]; ok {
drainAndClose(resp.Body, nil)
return err
}
}
if resp.StatusCode < 200 || resp.StatusCode > 299 {
drainAndClose(resp.Body, nil)
return newErrorf(resp.StatusCode, "HTTP Error: %d: %s", resp.StatusCode, resp.Status)
}
return nil
@ -324,7 +472,7 @@ again:
return
}
defer func() {
checkClose(resp.Body, &err)
drainAndClose(resp.Body, &err)
// Flush the auth connection - we don't want to keep
// it open if keepalives were enabled
flushKeepaliveConnections(c.Transport)
@ -447,7 +595,7 @@ func (c *Connection) QueryInfo() (infos SwiftInfo, err error) {
resp, err := c.client.Get(infoUrl.String())
if err == nil {
if resp.StatusCode != http.StatusOK {
resp.Body.Close()
drainAndClose(resp.Body, nil)
return nil, fmt.Errorf("Invalid status code for info request: %d", resp.StatusCode)
}
err = readJson(resp, &infos)
@ -497,6 +645,7 @@ type RequestOpts struct {
// Any other parameters (if not None) are added to the targetUrl
//
// Returns a response or an error. If response is returned then
// the resp.Body must be read completely and
// resp.Body.Close() must be called on it, unless noResponse is set in
// which case the body will be closed in this function
//
@ -571,7 +720,7 @@ func (c *Connection) Call(targetUrl string, p RequestOpts) (resp *http.Response,
}
// Check to see if token has expired
if resp.StatusCode == 401 && retries > 0 {
_ = resp.Body.Close()
drainAndClose(resp.Body, nil)
c.UnAuthenticate()
retries--
} else {
@ -580,12 +729,12 @@ func (c *Connection) Call(targetUrl string, p RequestOpts) (resp *http.Response,
}
if err = c.parseHeaders(resp, p.ErrorMap); err != nil {
_ = resp.Body.Close()
return nil, nil, err
}
headers = readHeaders(resp)
if p.NoResponse {
err = resp.Body.Close()
var err error
drainAndClose(resp.Body, &err)
if err != nil {
return nil, nil, err
}
@ -627,7 +776,7 @@ func (c *Connection) storage(p RequestOpts) (resp *http.Response, headers Header
//
// Closes the response when done
func readLines(resp *http.Response) (lines []string, err error) {
defer checkClose(resp.Body, &err)
defer drainAndClose(resp.Body, &err)
reader := bufio.NewReader(resp.Body)
buffer := bytes.NewBuffer(make([]byte, 0, 128))
var part []byte
@ -652,7 +801,7 @@ func readLines(resp *http.Response) (lines []string, err error) {
//
// Closes the response when done
func readJson(resp *http.Response, result interface{}) (err error) {
defer checkClose(resp.Body, &err)
defer drainAndClose(resp.Body, &err)
decoder := json.NewDecoder(resp.Body)
return decoder.Decode(result)
}

204
vendor/github.com/ncw/swift/swift_internal_test.go generated vendored
View file

@ -10,7 +10,10 @@ import (
"io"
"net"
"net/http"
"os"
"reflect"
"testing"
"time"
)
const (
@ -214,9 +217,9 @@ func TestInternalError(t *testing.T) {
}
func testCheckClose(c io.Closer, e error) (err error) {
func testCheckClose(rd io.ReadCloser, e error) (err error) {
err = e
defer checkClose(c, &err)
defer checkClose(rd, &err)
return
}
@ -225,6 +228,10 @@ type myCloser struct {
err error
}
func (c *myCloser) Read([]byte) (int, error) {
return 0, io.EOF
}
func (c *myCloser) Close() error {
return c.err
}
@ -426,3 +433,196 @@ func TestInternalObjectPutString(t *testing.T) {
defer server.Finished()
c.ObjectPutString("container", "object", "12345", "text/plain")
}
func TestSetFromEnv(t *testing.T) {
// String
s := ""
os.Setenv("POTATO", "")
err := setFromEnv(&s, "POTATO")
if err != nil {
t.Fatal(err)
}
os.Setenv("POTATO", "this is a test")
err = setFromEnv(&s, "POTATO")
if err != nil {
t.Fatal(err)
}
if s != "this is a test" {
t.Fatal("incorrect", s)
}
os.Setenv("POTATO", "new")
err = setFromEnv(&s, "POTATO")
if err != nil {
t.Fatal(err)
}
if s != "this is a test" {
t.Fatal("was reset when it shouldn't have been")
}
// Integer
i := 0
os.Setenv("POTATO", "42")
err = setFromEnv(&i, "POTATO")
if err != nil {
t.Fatal(err)
}
if i != 42 {
t.Fatal("incorrect", i)
}
os.Setenv("POTATO", "43")
err = setFromEnv(&i, "POTATO")
if err != nil {
t.Fatal(err)
}
if i != 42 {
t.Fatal("was reset when it shouldn't have been")
}
i = 0
os.Setenv("POTATO", "not a number")
err = setFromEnv(&i, "POTATO")
if err == nil {
t.Fatal("expecting error but didn't get one")
}
// bool
var b bool
os.Setenv("POTATO", "1")
err = setFromEnv(&b, "POTATO")
if err != nil {
t.Fatal(err)
}
if b != true {
t.Fatal("incorrect", b)
}
// time.Duration
var dt time.Duration
os.Setenv("POTATO", "5s")
err = setFromEnv(&dt, "POTATO")
if err != nil {
t.Fatal(err)
}
if dt != 5*time.Second {
t.Fatal("incorrect", dt)
}
// EndpointType
var e EndpointType
os.Setenv("POTATO", "internal")
err = setFromEnv(&e, "POTATO")
if err != nil {
t.Fatal(err)
}
if e != EndpointType("internal") {
t.Fatal("incorrect", e)
}
// Unknown
var unknown struct{}
err = setFromEnv(&unknown, "POTATO")
if err == nil {
t.Fatal("expecting error")
}
os.Setenv("POTATO", "")
}
func TestApplyEnvironment(t *testing.T) {
// We've tested all the setting logic above, so just do a quick test here
c := new(Connection)
os.Setenv("GOSWIFT_CONNECT_TIMEOUT", "100s")
err := c.ApplyEnvironment()
if err != nil {
t.Fatal(err)
}
if c.ConnectTimeout != 100*time.Second {
t.Fatal("timeout incorrect", c.ConnectTimeout)
}
c.ConnectTimeout = 0
os.Setenv("GOSWIFT_CONNECT_TIMEOUT", "parse error")
err = c.ApplyEnvironment()
if err == nil {
t.Fatal("expecting error")
}
if c.ConnectTimeout != 0 {
t.Fatal("timeout incorrect", c.ConnectTimeout)
}
os.Setenv("GOSWIFT_CONNECT_TIMEOUT", "")
}
func TestApplyEnvironmentAll(t *testing.T) {
// we do this in two phases because some of the variable set the same thing
for phase := 1; phase <= 2; phase++ {
c := new(Connection)
items := []struct {
phase int
result interface{}
name string
value string
want interface{}
oldValue string
}{
// Copied and amended from ApplyEnvironment
// Environment variables - keep in same order as Connection
{1, &c.Domain, "OS_USER_DOMAIN_NAME", "os_user_domain_name", "os_user_domain_name", ""},
{1, &c.DomainId, "OS_USER_DOMAIN_ID", "os_user_domain_id", "os_user_domain_id", ""},
{1, &c.UserName, "OS_USERNAME", "os_username", "os_username", ""},
{1, &c.ApiKey, "OS_PASSWORD", "os_password", "os_password", ""},
{1, &c.AuthUrl, "OS_AUTH_URL", "os_auth_url", "os_auth_url", ""},
{1, &c.Retries, "GOSWIFT_RETRIES", "4", 4, ""},
{1, &c.UserAgent, "GOSWIFT_USER_AGENT", "goswift_user_agent", "goswift_user_agent", ""},
{1, &c.ConnectTimeout, "GOSWIFT_CONNECT_TIMEOUT", "98s", 98 * time.Second, ""},
{1, &c.Timeout, "GOSWIFT_TIMEOUT", "99s", 99 * time.Second, ""},
{1, &c.Region, "OS_REGION_NAME", "os_region_name", "os_region_name", ""},
{1, &c.AuthVersion, "ST_AUTH_VERSION", "3", 3, ""},
{1, &c.Internal, "GOSWIFT_INTERNAL", "true", true, ""},
{1, &c.Tenant, "OS_TENANT_NAME", "os_tenant_name", "os_tenant_name", ""},
{2, &c.Tenant, "OS_PROJECT_NAME", "os_project_name", "os_project_name", ""},
{1, &c.TenantId, "OS_TENANT_ID", "os_tenant_id", "os_tenant_id", ""},
{1, &c.EndpointType, "OS_ENDPOINT_TYPE", "internal", EndpointTypeInternal, ""},
{1, &c.TenantDomain, "OS_PROJECT_DOMAIN_NAME", "os_project_domain_name", "os_project_domain_name", ""},
{1, &c.TenantDomainId, "OS_PROJECT_DOMAIN_ID", "os_project_domain_id", "os_project_domain_id", ""},
{1, &c.TrustId, "OS_TRUST_ID", "os_trust_id", "os_trust_id", ""},
{1, &c.StorageUrl, "OS_STORAGE_URL", "os_storage_url", "os_storage_url", ""},
{1, &c.AuthToken, "OS_AUTH_TOKEN", "os_auth_token", "os_auth_token", ""},
// v1 auth alternatives
{2, &c.ApiKey, "ST_KEY", "st_key", "st_key", ""},
{2, &c.UserName, "ST_USER", "st_user", "st_user", ""},
{2, &c.AuthUrl, "ST_AUTH", "st_auth", "st_auth", ""},
}
for i := range items {
item := &items[i]
if item.phase == phase {
item.oldValue = os.Getenv(item.name) // save old value
os.Setenv(item.name, item.value) // set new value
}
}
err := c.ApplyEnvironment()
if err != nil {
t.Fatalf("unexpected error %v", err)
}
for i := range items {
item := &items[i]
if item.phase == phase {
got := reflect.Indirect(reflect.ValueOf(item.result)).Interface()
if !reflect.DeepEqual(item.want, got) {
t.Errorf("%s: %v != %v", item.name, item.want, got)
}
os.Setenv(item.name, item.oldValue) // restore old value
}
}
}
}

33
vendor/github.com/ncw/swift/swift_test.go generated vendored
View file

@ -75,6 +75,7 @@ func makeConnection(t *testing.T) (*swift.Connection, func()) {
ConnectionChannelTimeout := os.Getenv("SWIFT_CONNECTION_CHANNEL_TIMEOUT")
DataChannelTimeout := os.Getenv("SWIFT_DATA_CHANNEL_TIMEOUT")
internalServer := false
if UserName == "" || ApiKey == "" || AuthUrl == "" {
srv, err = swifttest.NewSwiftServer("localhost")
if err != nil && t != nil {
@ -84,6 +85,7 @@ func makeConnection(t *testing.T) (*swift.Connection, func()) {
UserName = "swifttest"
ApiKey = "swifttest"
AuthUrl = srv.AuthURL
internalServer = true
}
transport := &http.Transport{
@ -105,6 +107,16 @@ func makeConnection(t *testing.T) (*swift.Connection, func()) {
EndpointType: swift.EndpointType(EndpointType),
}
if !internalServer {
if isV3Api() {
c.Tenant = os.Getenv("SWIFT_TENANT")
c.Domain = os.Getenv("SWIFT_API_DOMAIN")
} else {
c.Tenant = os.Getenv("SWIFT_TENANT")
c.TenantId = os.Getenv("SWIFT_TENANT_ID")
}
}
var timeout int64
if ConnectionChannelTimeout != "" {
timeout, err = strconv.ParseInt(ConnectionChannelTimeout, 10, 32)
@ -304,14 +316,6 @@ func TestTransport(t *testing.T) {
c.Transport = tr
if isV3Api() {
c.Tenant = os.Getenv("SWIFT_TENANT")
c.Domain = os.Getenv("SWIFT_API_DOMAIN")
} else {
c.Tenant = os.Getenv("SWIFT_TENANT")
c.TenantId = os.Getenv("SWIFT_TENANT_ID")
}
err := c.Authenticate()
if err != nil {
t.Fatal("Auth failed", err)
@ -329,9 +333,6 @@ func TestV1V2Authenticate(t *testing.T) {
c, rollback := makeConnection(t)
defer rollback()
c.Tenant = os.Getenv("SWIFT_TENANT")
c.TenantId = os.Getenv("SWIFT_TENANT_ID")
err := c.Authenticate()
if err != nil {
t.Fatal("Auth failed", err)
@ -349,8 +350,10 @@ func TestV3AuthenticateWithDomainNameAndTenantId(t *testing.T) {
c, rollback := makeConnection(t)
defer rollback()
c.TenantId = os.Getenv("SWIFT_TENANT_ID")
c.Tenant = ""
c.Domain = os.Getenv("SWIFT_API_DOMAIN")
c.TenantId = os.Getenv("SWIFT_TENANT_ID")
c.DomainId = ""
err := c.Authenticate()
if err != nil {
@ -388,6 +391,8 @@ func TestV3AuthenticateWithDomainIdAndTenantId(t *testing.T) {
c, rollback := makeConnection(t)
defer rollback()
c.Tenant = ""
c.Domain = ""
c.TenantId = os.Getenv("SWIFT_TENANT_ID")
c.DomainId = os.Getenv("SWIFT_API_DOMAIN_ID")
@ -410,6 +415,8 @@ func TestV3AuthenticateWithDomainNameAndTenantName(t *testing.T) {
c.Tenant = os.Getenv("SWIFT_TENANT")
c.Domain = os.Getenv("SWIFT_API_DOMAIN")
c.TenantId = ""
c.DomainId = ""
err := c.Authenticate()
if err != nil {
@ -429,6 +436,8 @@ func TestV3AuthenticateWithDomainIdAndTenantName(t *testing.T) {
defer rollback()
c.Tenant = os.Getenv("SWIFT_TENANT")
c.Domain = ""
c.TenantId = ""
c.DomainId = os.Getenv("SWIFT_API_DOMAIN_ID")
err := c.Authenticate()