s3: Support Alibaba Cloud (Aliyun) OSS

The existing s3 backend passed all integration tests with OSS provided `force_path_style = false`. This makes sure that is so and adds documentation and configuration for OSS. Thanks to @luolibin for their work on the OSS backend which we ended up not needing. Fixes #1641 Fixes #1237
2019-01-12 16:46:45 +00:00 · 2019-01-12 16:46:45 +00:00 · ff0b8e10af
commit ff0b8e10af
parent 8d16a5693c
5 changed files with 230 additions and 65 deletions
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@ -372,4 +372,3 @@ Add your fs to the docs - you'll need to pick an icon for it from [fontawesome](
  * `docs/content/about.md` - front page of rclone.org
  * `docs/layouts/chrome/navbar.html` - add it to the website navigation
  * `bin/make_manual.py` - add the page to the `docs` constant
-  * `cmd/cmd.go` - the main help for rclone
--- a/README.md
+++ b/README.md
@ -20,6 +20,7 @@ Rclone *("rsync for cloud storage")* is a command line program to sync files and

 ## Storage providers

+  * Alibaba Cloud (Aliyun) Object Storage System (OSS) [:page_facing_up:](https://rclone.org/s3/#alibaba-oss)
  * Amazon Drive [:page_facing_up:](https://rclone.org/amazonclouddrive/) ([See note](https://rclone.org/amazonclouddrive/#status))
  * Amazon S3 [:page_facing_up:](https://rclone.org/s3/)
  * Backblaze B2 [:page_facing_up:](https://rclone.org/b2/)
--- a/backend/s3/s3.go
+++ b/backend/s3/s3.go
@ -53,7 +53,7 @@ import (
 func init() {
 	fs.Register(&fs.RegInfo{
 		Name:        "s3",
-		Description: "Amazon S3 Compliant Storage Providers (AWS, Ceph, Dreamhost, IBM COS, Minio)",
+		Description: "Amazon S3 Compliant Storage Provider (AWS, Alibaba, Ceph, Digital Ocean, Dreamhost, IBM COS, Minio, etc)",
 		NewFs:       NewFs,
 		Options: []fs.Option{{
 			Name: fs.ConfigProvider,
@ -61,6 +61,9 @@ func init() {
 			Examples: []fs.OptionExample{{
 				Value: "AWS",
 				Help:  "Amazon Web Services (AWS) S3",
+			}, {
+				Value: "Alibaba",
+				Help:  "Alibaba Cloud Object Storage System (OSS) formerly Aliyun",
 			}, {
 				Value: "Ceph",
 				Help:  "Ceph Object Storage",
@ -76,6 +79,9 @@ func init() {
 			}, {
 				Value: "Minio",
 				Help:  "Minio Object Storage",
+			}, {
+				Value: "Netease",
+				Help:  "Netease Object Storage (NOS)",
 			}, {
 				Value: "Wasabi",
 				Help:  "Wasabi Object Storage",
@ -150,7 +156,7 @@ func init() {
 		}, {
 			Name:     "region",
 			Help:     "Region to connect to.\nLeave blank if you are using an S3 clone and you don't have a region.",
-			Provider: "!AWS",
+			Provider: "!AWS,Alibaba",
 			Examples: []fs.OptionExample{{
 				Value: "",
 				Help:  "Use this if unsure. Will use v4 signatures and an empty region.",
@ -269,10 +275,73 @@ func init() {
 				Value: "s3.tor01.objectstorage.service.networklayer.com",
 				Help:  "Toronto Single Site Private Endpoint",
 			}},
+		}, {
+			// oss endpoints: https://help.aliyun.com/document_detail/31837.html
+			Name:     "endpoint",
+			Help:     "Endpoint for OSS API.",
+			Provider: "Alibaba",
+			Examples: []fs.OptionExample{{
+				Value: "oss-cn-hangzhou.aliyuncs.com",
+				Help:  "East China 1 (Hangzhou)",
+			}, {
+				Value: "oss-cn-shanghai.aliyuncs.com",
+				Help:  "East China 2 (Shanghai)",
+			}, {
+				Value: "oss-cn-qingdao.aliyuncs.com",
+				Help:  "North China 1 (Qingdao)",
+			}, {
+				Value: "oss-cn-beijing.aliyuncs.com",
+				Help:  "North China 2 (Beijing)",
+			}, {
+				Value: "oss-cn-zhangjiakou.aliyuncs.com",
+				Help:  "North China 3 (Zhangjiakou)",
+			}, {
+				Value: "oss-cn-huhehaote.aliyuncs.com",
+				Help:  "North China 5 (Huhehaote)",
+			}, {
+				Value: "oss-cn-shenzhen.aliyuncs.com",
+				Help:  "South China 1 (Shenzhen)",
+			}, {
+				Value: "oss-cn-hongkong.aliyuncs.com",
+				Help:  "Hong Kong (Hong Kong)",
+			}, {
+				Value: "oss-us-west-1.aliyuncs.com",
+				Help:  "US West 1 (Silicon Valley)",
+			}, {
+				Value: "oss-us-east-1.aliyuncs.com",
+				Help:  "US East 1 (Virginia)",
+			}, {
+				Value: "oss-ap-southeast-1.aliyuncs.com",
+				Help:  "Southeast Asia Southeast 1 (Singapore)",
+			}, {
+				Value: "oss-ap-southeast-2.aliyuncs.com",
+				Help:  "Asia Pacific Southeast 2 (Sydney)",
+			}, {
+				Value: "oss-ap-southeast-3.aliyuncs.com",
+				Help:  "Southeast Asia Southeast 3 (Kuala Lumpur)",
+			}, {
+				Value: "oss-ap-southeast-5.aliyuncs.com",
+				Help:  "Asia Pacific Southeast 5 (Jakarta)",
+			}, {
+				Value: "oss-ap-northeast-1.aliyuncs.com",
+				Help:  "Asia Pacific Northeast 1 (Japan)",
+			}, {
+				Value: "oss-ap-south-1.aliyuncs.com",
+				Help:  "Asia Pacific South 1 (Mumbai)",
+			}, {
+				Value: "oss-eu-central-1.aliyuncs.com",
+				Help:  "Central Europe 1 (Frankfurt)",
+			}, {
+				Value: "oss-eu-west-1.aliyuncs.com",
+				Help:  "West Europe (London)",
+			}, {
+				Value: "oss-me-east-1.aliyuncs.com",
+				Help:  "Middle East 1 (Dubai)",
+			}},
 		}, {
 			Name:     "endpoint",
 			Help:     "Endpoint for S3 API.\nRequired when using an S3 clone.",
-			Provider: "!AWS,IBMCOS",
+			Provider: "!AWS,IBMCOS,Alibaba",
 			Examples: []fs.OptionExample{{
 				Value:    "objects-us-west-1.dream.io",
 				Help:     "Dream Objects endpoint",
@ -449,7 +518,7 @@ func init() {
 		}, {
 			Name:     "location_constraint",
 			Help:     "Location constraint - must be set to match the Region.\nLeave blank if not sure. Used when creating buckets only.",
-			Provider: "!AWS,IBMCOS",
+			Provider: "!AWS,IBMCOS,Alibaba",
 		}, {
 			Name: "acl",
 			Help: `Canned ACL used when creating buckets and storing or copying objects.
@ -547,6 +616,20 @@ doesn't copy the ACL from the source but rather writes a fresh one.`,
 				Value: "GLACIER",
 				Help:  "Glacier storage class",
 			}},
+		}, {
+			Name:     "storage_class",
+			Help:     "The storage class to use when storing new objects in OSS.",
+			Provider: "Alibaba",
+			Examples: []fs.OptionExample{{
+				Value: "Standard",
+				Help:  "Standard storage class",
+			}, {
+				Value: "Archive",
+				Help:  "Archive storage mode.",
+			}, {
+				Value: "IA",
+				Help:  "Infrequent access storage mode.",
+			}},
 		}, {
 			Name: "upload_cutoff",
 			Help: `Cutoff for switching to chunked upload
@ -714,7 +797,7 @@ func (f *Fs) Features() *fs.Features {
 // retryErrorCodes is a slice of error codes that we will retry
 // See: https://docs.aws.amazon.com/AmazonS3/latest/API/ErrorResponses.html
 var retryErrorCodes = []int{
-	409, // Conflict - various states that could be resolved on a retry
+	// 409, // Conflict - various states that could be resolved on a retry
 	503, // Service Unavailable/Slow Down - "Reduce your request rate"
 }

@ -722,14 +805,13 @@ var retryErrorCodes = []int{
 // as it should notice closed connections and timeouts which are the most likely
 // sort of failure modes
 func shouldRetry(err error) (bool, error) {
-
 	// If this is an awserr object, try and extract more useful information to determine if we should retry
 	if awsError, ok := err.(awserr.Error); ok {
 		// Simple case, check the original embedded error in case it's generically retriable
 		if fserrors.ShouldRetry(awsError.OrigErr()) {
 			return true, err
 		}
-		//Failing that, if it's a RequestFailure it's probably got an http status code we can check
+		// Failing that, if it's a RequestFailure it's probably got an http status code we can check
 		if reqErr, ok := err.(awserr.RequestFailure); ok {
 			for _, e := range retryErrorCodes {
 				if reqErr.StatusCode() == e {
@ -738,7 +820,7 @@ func shouldRetry(err error) (bool, error) {
 			}
 		}
 	}
-	//Ok, not an awserr, check for generic failure conditions
+	// Ok, not an awserr, check for generic failure conditions
 	return fserrors.ShouldRetry(err), err
 }

@ -815,6 +897,9 @@ func s3Connection(opt *Options) (*s3.S3, *session.Session, error) {
 	if opt.Region == "" {
 		opt.Region = "us-east-1"
 	}
+	if opt.Provider == "Alibaba" || opt.Provider == "Netease" {
+		opt.ForcePathStyle = false
+	}
 	awsConfig := aws.NewConfig().
 		WithRegion(opt.Region).
 		WithMaxRetries(maxRetries).
--- a/docs/content/about.md
+++ b/docs/content/about.md
@ -13,6 +13,7 @@ Rclone

 Rclone is a command line program to sync files and directories to and from:

+* {{< provider name="Alibaba Cloud (Aliyun) Object Storage System (OSS)" home="https://www.alibabacloud.com/product/oss/" config="/s3/#alibaba-oss" >}}
 * {{< provider name="Amazon Drive" home="https://www.amazon.com/clouddrive" config="/amazonclouddrive/" >}} ([See note](/amazonclouddrive/#status))
 * {{< provider name="Amazon S3" home="https://aws.amazon.com/s3/" config="/s3/" >}}
 * {{< provider name="Backblaze B2" home="https://www.backblaze.com/b2/cloud-storage.html" config="/b2/" >}}
--- a/docs/content/s3.md
+++ b/docs/content/s3.md
@ -10,6 +10,7 @@ date: "2016-07-11"
 The S3 backend can be used with a number of different providers:

 * {{< provider name="AWS S3" home="https://aws.amazon.com/s3/" config="/s3/#amazon-s3" >}}
+* {{< provider name="Alibaba Cloud (Aliyun) Object Storage System (OSS)" home="https://www.alibabacloud.com/product/oss/" config="/s3/#alibaba-oss" >}}
 * {{< provider name="Ceph" home="http://ceph.com/" config="/s3/#ceph" >}}
 * {{< provider name="DigitalOcean Spaces" home="https://www.digitalocean.com/products/object-storage/" config="/s3/#digitalocean-spaces" >}}
 * {{< provider name="Dreamhost" home="https://www.dreamhost.com/cloud/storage/" config="/s3/#dreamhost" >}}
@ -400,7 +401,7 @@ the object(s) in question before using rclone.
 <!--- autogenerated options start - DO NOT EDIT, instead edit fs.RegInfo in backend/s3/s3.go then run make backenddocs -->
 ### Standard Options

-Here are the standard options specific to s3 (Amazon S3 Compliant Storage Providers (AWS, Ceph, Dreamhost, IBM COS, Minio)).
+Here are the standard options specific to s3 (Amazon S3 Compliant Storage Provider (AWS, Alibaba, Ceph, Digital Ocean, Dreamhost, IBM COS, Minio, etc)).

 #### --s3-provider

@ -413,6 +414,8 @@ Choose your S3 provider.
 - Examples:
    - "AWS"
        - Amazon Web Services (AWS) S3
+    - "Alibaba"
+        - Alibaba Cloud Object Storage System (OSS) formerly Aliyun
    - "Ceph"
        - Ceph Object Storage
    - "DigitalOcean"
@ -423,6 +426,8 @@ Choose your S3 provider.
        - IBM COS S3
    - "Minio"
        - Minio Object Storage
+    - "Netease"
+        - Netease Object Storage (NOS)
    - "Wasabi"
        - Wasabi Object Storage
    - "Other"
@ -622,6 +627,54 @@ Specify if using an IBM COS On Premise.

 #### --s3-endpoint

+Endpoint for OSS API.
+
+- Config:      endpoint
+- Env Var:     RCLONE_S3_ENDPOINT
+- Type:        string
+- Default:     ""
+- Examples:
+    - "oss-cn-hangzhou.aliyuncs.com"
+        - East China 1 (Hangzhou)
+    - "oss-cn-shanghai.aliyuncs.com"
+        - East China 2 (Shanghai)
+    - "oss-cn-qingdao.aliyuncs.com"
+        - North China 1 (Qingdao)
+    - "oss-cn-beijing.aliyuncs.com"
+        - North China 2 (Beijing)
+    - "oss-cn-zhangjiakou.aliyuncs.com"
+        - North China 3 (Zhangjiakou)
+    - "oss-cn-huhehaote.aliyuncs.com"
+        - North China 5 (Huhehaote)
+    - "oss-cn-shenzhen.aliyuncs.com"
+        - South China 1 (Shenzhen)
+    - "oss-cn-hongkong.aliyuncs.com"
+        - Hong Kong (Hong Kong)
+    - "oss-us-west-1.aliyuncs.com"
+        - US West 1 (Silicon Valley)
+    - "oss-us-east-1.aliyuncs.com"
+        - US East 1 (Virginia)
+    - "oss-ap-southeast-1.aliyuncs.com"
+        - Southeast Asia Southeast 1 (Singapore)
+    - "oss-ap-southeast-2.aliyuncs.com"
+        - Asia Pacific Southeast 2 (Sydney)
+    - "oss-ap-southeast-3.aliyuncs.com"
+        - Southeast Asia Southeast 3 (Kuala Lumpur)
+    - "oss-ap-southeast-5.aliyuncs.com"
+        - Asia Pacific Southeast 5 (Jakarta)
+    - "oss-ap-northeast-1.aliyuncs.com"
+        - Asia Pacific Northeast 1 (Japan)
+    - "oss-ap-south-1.aliyuncs.com"
+        - Asia Pacific South 1 (Mumbai)
+    - "oss-eu-central-1.aliyuncs.com"
+        - Central Europe 1 (Frankfurt)
+    - "oss-eu-west-1.aliyuncs.com"
+        - West Europe (London)
+    - "oss-me-east-1.aliyuncs.com"
+        - Middle East 1 (Dubai)
+
+#### --s3-endpoint
+
 Endpoint for S3 API.
 Required when using an S3 clone.

@ -855,11 +908,27 @@ The storage class to use when storing new objects in S3.
    - "ONEZONE_IA"
        - One Zone Infrequent Access storage class
    - "GLACIER"
-        - GLACIER storage class
+        - Glacier storage class
+
+#### --s3-storage-class
+
+The storage class to use when storing new objects in OSS.
+
+- Config:      storage_class
+- Env Var:     RCLONE_S3_STORAGE_CLASS
+- Type:        string
+- Default:     ""
+- Examples:
+    - "Standard"
+        - Standard storage class
+    - "Archive"
+        - Archive storage mode.
+    - "IA"
+        - Infrequent access storage mode.

 ### Advanced Options

-Here are the advanced options specific to s3 (Amazon S3 Compliant Storage Providers (AWS, Ceph, Dreamhost, IBM COS, Minio)).
+Here are the advanced options specific to s3 (Amazon S3 Compliant Storage Provider (AWS, Alibaba, Ceph, Digital Ocean, Dreamhost, IBM COS, Minio, etc)).

 #### --s3-upload-cutoff

@ -1458,30 +1527,41 @@ server_side_encryption =
 storage_class =
 ```

-### Aliyun OSS / Netease NOS  ###
+### Alibaba OSS {#alibaba-oss}

-This describes how to set up Aliyun OSS - Netease NOS is the same
-except for different endpoints.
+Here is an example of making an [Alibaba Cloud (Aliyun) OSS](https://www.alibabacloud.com/product/oss/)
+configuration.  First run:

-Note this is a pretty standard S3 setup, except for the setting of
-`force_path_style = false` in the advanced config.
+    rclone config
+
+This will guide you through an interactive setup process.

 ```
-# rclone config
-e/n/d/r/c/s/q> n
+No remotes found - make a new one
+n) New remote
+s) Set configuration password
+q) Quit config
+n/s/q> n
 name> oss
 Type of storage to configure.
 Enter a string value. Press Enter for the default ("").
 Choose a number from below, or type in your own value
- 3 / Amazon S3 Compliant Storage Providers (AWS, Ceph, Dreamhost, IBM COS, Minio)
+[snip]
+ 4 / Amazon S3 Compliant Storage Provider (AWS, Alibaba, Ceph, Digital Ocean, Dreamhost, IBM COS, Minio, etc)
   \ "s3"
+[snip]
 Storage> s3
 Choose your S3 provider.
 Enter a string value. Press Enter for the default ("").
 Choose a number from below, or type in your own value
- 8 / Any other S3 compatible provider
-   \ "Other"
-provider> other
+ 1 / Amazon Web Services (AWS) S3
+   \ "AWS"
+ 2 / Alibaba Cloud Object Storage System (OSS) formerly Aliyun
+   \ "Alibaba"
+ 3 / Ceph Object Storage
+   \ "Ceph"
+[snip]
+provider> Alibaba
 Get AWS credentials from runtime (environment variables or EC2/ECS meta data if no env vars).
 Only applies if access_key_id and secret_access_key is blank.
 Enter a boolean value (true or false). Press Enter for the default ("false").
@ -1494,70 +1574,69 @@ env_auth> 1
 AWS Access Key ID.
 Leave blank for anonymous access or runtime credentials.
 Enter a string value. Press Enter for the default ("").
-access_key_id> xxxxxxxxxxxx
+access_key_id> accesskeyid
 AWS Secret Access Key (password)
 Leave blank for anonymous access or runtime credentials.
 Enter a string value. Press Enter for the default ("").
-secret_access_key> xxxxxxxxxxxxxxxxx
-Region to connect to.
-Leave blank if you are using an S3 clone and you don't have a region.
+secret_access_key> secretaccesskey
+Endpoint for OSS API.
 Enter a string value. Press Enter for the default ("").
 Choose a number from below, or type in your own value
- 1 / Use this if unsure. Will use v4 signatures and an empty region.
-   \ ""
- 2 / Use this only if v4 signatures don't work, eg pre Jewel/v10 CEPH.
-   \ "other-v2-signature"
-region> 1
-Endpoint for S3 API.
-Required when using an S3 clone.
-Enter a string value. Press Enter for the default ("").
-Choose a number from below, or type in your own value
-endpoint> oss-cn-shenzhen.aliyuncs.com
-Location constraint - must be set to match the Region.
-Leave blank if not sure. Used when creating buckets only.
-Enter a string value. Press Enter for the default ("").
-location_constraint>
-Canned ACL used when creating buckets and/or storing objects in S3.
-For more info visit https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl
+ 1 / East China 1 (Hangzhou)
+   \ "oss-cn-hangzhou.aliyuncs.com"
+ 2 / East China 2 (Shanghai)
+   \ "oss-cn-shanghai.aliyuncs.com"
+ 3 / North China 1 (Qingdao)
+   \ "oss-cn-qingdao.aliyuncs.com"
+[snip]
+endpoint> 1
+Canned ACL used when creating buckets and storing or copying objects.
+
+Note that this ACL is applied when server side copying objects as S3
+doesn't copy the ACL from the source but rather writes a fresh one.
 Enter a string value. Press Enter for the default ("").
 Choose a number from below, or type in your own value
 1 / Owner gets FULL_CONTROL. No one else has access rights (default).
   \ "private"
+ 2 / Owner gets FULL_CONTROL. The AllUsers group gets READ access.
+   \ "public-read"
+   / Owner gets FULL_CONTROL. The AllUsers group gets READ and WRITE access.
+[snip]
 acl> 1
+The storage class to use when storing new objects in OSS.
+Enter a string value. Press Enter for the default ("").
+Choose a number from below, or type in your own value
+ 1 / Standard storage class
+   \ "Standard"
+ 2 / Archive storage mode.
+   \ "Archive"
+ 3 / Infrequent access storage mode.
+   \ "IA"
+storage_class> 1
 Edit advanced config? (y/n)
 y) Yes
 n) No
-y/n> y
-Chunk size to use for uploading
-Enter a size with suffix k,M,G,T. Press Enter for the default ("5M").
-chunk_size>
-Don't store MD5 checksum with object metadata
-Enter a boolean value (true or false). Press Enter for the default ("false").
-disable_checksum>
-An AWS session token
-Enter a string value. Press Enter for the default ("").
-session_token>
-Concurrency for multipart uploads.
-Enter a signed integer. Press Enter for the default ("2").
-upload_concurrency>
-If true use path style access if false use virtual hosted style.
-Some providers (eg Aliyun OSS or Netease COS) require this.
-Enter a boolean value (true or false). Press Enter for the default ("true").
-force_path_style> false
+y/n> n
 Remote config
 --------------------
 [oss]
 type = s3
-provider = Other
+provider = Alibaba
 env_auth = false
-access_key_id = xxxxxxxxx
-secret_access_key = xxxxxxxxxxxxx
-endpoint = oss-cn-shenzhen.aliyuncs.com
+access_key_id = accesskeyid
+secret_access_key = secretaccesskey
+endpoint = oss-cn-hangzhou.aliyuncs.com
 acl = private
-force_path_style = false
+storage_class = Standard
 --------------------
 y) Yes this is OK
 e) Edit this remote
 d) Delete this remote
 y/e/d> y
 ```
+
+### Netease NOS  ###
+
+For Netease NOS configure as per the configurator `rclone config`
+setting the provider `Netease`.  This will automatically set
+`force_path_style = false` which is necessary for it to run properly.