From 9c47b767b4afe9fa2681cecb140c834dfa750644 Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Thu, 10 Aug 2017 21:38:45 +0100 Subject: [PATCH] swift: Configure from environment vars and add endpoint_type - fixes #1542 --- docs/content/swift.md | 81 ++++++++++++++++++++++++++++++++++++------- swift/swift.go | 60 +++++++++++++++++++++++--------- 2 files changed, 112 insertions(+), 29 deletions(-) diff --git a/docs/content/swift.md b/docs/content/swift.md index e95572c93..2b68bf02d 100644 --- a/docs/content/swift.md +++ b/docs/content/swift.md @@ -7,7 +7,7 @@ date: "2014-04-26" Swift ---------------------------------------- -Swift refers to [Openstack Object Storage](https://www.openstack.org/software/openstack-storage/). +Swift refers to [Openstack Object Storage](https://docs.openstack.org/swift/latest/). Commercial implementations of that being: * [Rackspace Cloud Files](https://www.rackspace.com/cloud/files/) @@ -26,7 +26,8 @@ This will guide you through an interactive setup process. No remotes found - make a new one n) New remote s) Set configuration password -n/s> n +q) Quit config +n/s/q> n name> remote Type of storage to configure. Choose a number from below, or type in your own value @@ -36,27 +37,44 @@ Choose a number from below, or type in your own value \ "s3" 3 / Backblaze B2 \ "b2" - 4 / Dropbox + 4 / Box + \ "box" + 5 / Dropbox \ "dropbox" - 5 / Encrypt/Decrypt a remote + 6 / Encrypt/Decrypt a remote \ "crypt" - 6 / Google Cloud Storage (this is not Google Drive) + 7 / FTP Connection + \ "ftp" + 8 / Google Cloud Storage (this is not Google Drive) \ "google cloud storage" - 7 / Google Drive + 9 / Google Drive \ "drive" - 8 / Hubic +10 / Hubic \ "hubic" - 9 / Local Disk +11 / Local Disk \ "local" -10 / Microsoft OneDrive +12 / Microsoft Azure Blob Storage + \ "azureblob" +13 / Microsoft OneDrive \ "onedrive" -11 / Openstack Swift (Rackspace Cloud Files, Memset Memstore, OVH) +14 / Openstack Swift (Rackspace Cloud Files, Memset Memstore, OVH) \ "swift" -12 / SSH/SFTP Connection +15 / QingClound Object Storage + \ "qingstor" +16 / SSH/SFTP Connection \ "sftp" -13 / Yandex Disk +17 / Yandex Disk \ "yandex" -Storage> 11 +18 / http Connection + \ "http" +Storage> swift +Get swift credentials from environment variables in standard OpenStack form. +Choose a number from below, or type in your own value + 1 / Enter swift credentials in the next step + \ "false" + 2 / Get swift credentials from environment vars. Leave other fields blank if using this. + \ "true" +env_auth> 1 User name to log in. user> user_name API key or password. @@ -88,9 +106,19 @@ Storage URL - optional storage_url> AuthVersion - optional - set to (1,2,3) if your auth URL has no version auth_version> +Endpoint type to choose from the service catalogue +Choose a number from below, or type in your own value + 1 / Public (default, choose this if not sure) + \ "public" + 2 / Internal (use internal service net) + \ "internal" + 3 / Admin + \ "admin" +endpoint_type> Remote config -------------------- [remote] +env_auth = false user = user_name key = password_or_api_key auth = https://auth.api.rackspacecloud.com/v1.0 @@ -100,6 +128,7 @@ tenant_domain = region = storage_url = auth_version = +endpoint_type = -------------------- y) Yes this is OK e) Edit this remote @@ -158,6 +187,32 @@ tenant = $OS_TENANT_NAME Note that you may (or may not) need to set `region` too - try without first. +### Configuration from the environment ### + +If you prefer you can configure rclone to use swift using a standard +set of OpenStack environment variables. + +When you run through the config, make sure you choose `true` for +`env_auth` and leave everything else blank. + +rclone will then set any empty config parameters from the enviroment +using standard OpenStack environment variables. There is [a list of +the +variables](https://godoc.org/github.com/ncw/swift#Connection.ApplyEnvironment) +in the docs for the swift library. + +#### Using rclone without a config file #### + +You can use rclone with swift without a config file, if desired, like +this: + +``` +source openstack-credentials-file +export RCLONE_CONFIG_MYREMOTE_TYPE=swift +export RCLONE_CONFIG_MYREMOTE_ENV_AUTH=true +rclone lsd myremote: +``` + ### --fast-list ### This remote supports `--fast-list` which allows you to use fewer diff --git a/swift/swift.go b/swift/swift.go index 489402573..c71975e77 100644 --- a/swift/swift.go +++ b/swift/swift.go @@ -35,6 +35,18 @@ func init() { Description: "Openstack Swift (Rackspace Cloud Files, Memset Memstore, OVH)", NewFs: NewFs, Options: []fs.Option{{ + Name: "env_auth", + Help: "Get swift credentials from environment variables in standard OpenStack form.", + Examples: []fs.OptionExample{ + { + Value: "false", + Help: "Enter swift credentials in the next step", + }, { + Value: "true", + Help: "Get swift credentials from environment vars. Leave other fields blank if using this.", + }, + }, + }, { Name: "user", Help: "User name to log in.", }, { @@ -80,10 +92,22 @@ func init() { }, { Name: "auth_version", Help: "AuthVersion - optional - set to (1,2,3) if your auth URL has no version", + }, { + Name: "endpoint_type", + Help: "Endpoint type to choose from the service catalogue", + Examples: []fs.OptionExample{{ + Help: "Public (default, choose this if not sure)", + Value: "public", + }, { + Help: "Internal (use internal service net)", + Value: "internal", + }, { + Help: "Admin", + Value: "admin", + }}, }, }, }) - // snet = flag.Bool("swift-snet", false, "Use internal service network") // FIXME not implemented fs.VarP(&chunkSize, "swift-chunk-size", "", "Above this size files will be chunked into a _segments container.") } @@ -154,31 +178,35 @@ func parsePath(path string) (container, directory string, err error) { // swiftConnection makes a connection to swift func swiftConnection(name string) (*swift.Connection, error) { - userName := fs.ConfigFileGet(name, "user") - if userName == "" { - return nil, errors.New("user not found") - } - apiKey := fs.ConfigFileGet(name, "key") - if apiKey == "" { - return nil, errors.New("key not found") - } - authURL := fs.ConfigFileGet(name, "auth") - if authURL == "" { - return nil, errors.New("auth not found") - } c := &swift.Connection{ - UserName: userName, - ApiKey: apiKey, - AuthUrl: authURL, + UserName: fs.ConfigFileGet(name, "user"), + ApiKey: fs.ConfigFileGet(name, "key"), + AuthUrl: fs.ConfigFileGet(name, "auth"), AuthVersion: fs.ConfigFileGetInt(name, "auth_version", 0), Tenant: fs.ConfigFileGet(name, "tenant"), Region: fs.ConfigFileGet(name, "region"), Domain: fs.ConfigFileGet(name, "domain"), TenantDomain: fs.ConfigFileGet(name, "tenant_domain"), + EndpointType: swift.EndpointType(fs.ConfigFileGet(name, "endpoint_type", "public")), ConnectTimeout: 10 * fs.Config.ConnectTimeout, // Use the timeouts in the transport Timeout: 10 * fs.Config.Timeout, // Use the timeouts in the transport Transport: fs.Config.Transport(), } + if fs.ConfigFileGetBool(name, "env_auth", false) { + err := c.ApplyEnvironment() + if err != nil { + return nil, errors.Wrap(err, "failed to read environment variables") + } + } + if c.UserName == "" { + return nil, errors.New("user not found") + } + if c.ApiKey == "" { + return nil, errors.New("key not found") + } + if c.AuthUrl == "" { + return nil, errors.New("auth not found") + } err := c.Authenticate() if err != nil { return nil, err