From acd5d4377ecf6399ca75a25cb5933d0893efc8a3 Mon Sep 17 00:00:00 2001 From: Giri Badanahatti Date: Thu, 12 Apr 2018 11:05:53 -0500 Subject: [PATCH] config,s3: hierarchical configuration support #2140 This introduces a method of making provider specific configuration within a remote. This is useful particularly in s3. This commit does the basic configuration in S3 for IBM COS. --- backend/s3/s3.go | 581 +++++++++++++++++++++++++++++++----------- docs/content/about.md | 2 +- docs/content/s3.md | 301 +++++++++++----------- fs/config/config.go | 39 ++- fs/fs.go | 6 +- 5 files changed, 623 insertions(+), 306 deletions(-) diff --git a/backend/s3/s3.go b/backend/s3/s3.go index f26f63829..22f11d5d4 100644 --- a/backend/s3/s3.go +++ b/backend/s3/s3.go @@ -51,177 +51,447 @@ import ( func init() { fs.Register(&fs.RegInfo{ Name: "s3", - Description: "Amazon S3 (also Dreamhost, Ceph, Minio, IBM COS)", + Description: "Amazon S3 Compliant Storage Providers (AWS, Ceph, Dreamhost, IBM COS, Minio)", NewFs: NewFs, - // AWS endpoints: http://docs.amazonwebservices.com/general/latest/gr/rande.html#s3_region Options: []fs.Option{{ - Name: "env_auth", - Help: "Get AWS credentials from runtime (environment variables or EC2/ECS meta data if no env vars). Only applies if access_key_id and secret_access_key is blank.", + Name: "Provider", + Help: "Choose the S3 provider.", Examples: []fs.OptionExample{ { - Value: "false", - Help: "Enter AWS credentials in the next step", - }, { - Value: "true", - Help: "Get AWS credentials from the environment (env vars or IAM)", + Value: "AWS", + Help: "Choose this option to configure Storage to AWS S3", + }, + { + Value: "Ceph", + Help: "Choose this option to configure Storage to Ceph Systems", + }, + { + Value: "Dreamhost", + Help: " Choose this option to configure Storage to Dreamhost", + }, + { + Value: "IBMCOS", + Help: "Choose this option to the configure Storage to IBM COS S3", + }, + { + Value: "Minio", + Help: "Choose this option to the configure Storage to Minio", + }, + { + Value: "Other", + Help: "Choose this option to the configure any other S3 Storage", }, }, - }, { - Name: "access_key_id", - Help: "AWS Access Key ID - leave blank for anonymous access or runtime credentials.", - }, { - Name: "secret_access_key", - Help: "AWS Secret Access Key (password) - leave blank for anonymous access or runtime credentials.", - }, { - Name: "region", - Help: "Region to connect to. Leave blank if you are using an S3 clone and you don't have a region.", - Examples: []fs.OptionExample{{ - Value: "us-east-1", - Help: "The default endpoint - a good choice if you are unsure.\nUS Region, Northern Virginia or Pacific Northwest.\nLeave location constraint empty.", + }, + { + Name: "env_auth", + Help: "Get AWS credentials from runtime (environment variables or EC2/ECS meta data if no env vars). Only applies if access_key_id and secret_access_key is blank.", + Provider: "AWS,Ceph,Dreamhost,Minio,Other", + Examples: []fs.OptionExample{ + { + Value: "false", + Help: "Enter AWS credentials in the next step", + }, { + Value: "true", + Help: "Get AWS credentials from the environment (env vars or IAM)", + }, + }, }, { - Value: "us-east-2", - Help: "US East (Ohio) Region\nNeeds location constraint us-east-2.", + Name: "access_key_id", + Help: "AWS Access Key ID - leave blank for anonymous access or runtime credentials.", + Provider: "AWS,Ceph,Dreamhost,IBMCOS,Minio,Other", }, { - Value: "us-west-2", - Help: "US West (Oregon) Region\nNeeds location constraint us-west-2.", + Name: "secret_access_key", + Help: "AWS Secret Access Key (password) - leave blank for anonymous access or runtime credentials.", + Provider: "AWS,Ceph,Dreamhost,IBMCOS,Minio,Other", }, { - Value: "us-west-1", - Help: "US West (Northern California) Region\nNeeds location constraint us-west-1.", + Name: "region", + Help: "Region to connect to. Leave blank if you are using an S3 clone and you don't have a region.", + Provider: "AWS,Ceph,Dreamhost,Minio,Other", + Examples: []fs.OptionExample{{ + Value: "us-east-1", + Help: "The default endpoint - a good choice if you are unsure.\nUS Region, Northern Virginia or Pacific Northwest.\nLeave location constraint empty.", + }, { + Value: "us-east-2", + Help: "US East (Ohio) Region\nNeeds location constraint us-east-2.", + }, { + Value: "us-west-2", + Help: "US West (Oregon) Region\nNeeds location constraint us-west-2.", + }, { + Value: "us-west-1", + Help: "US West (Northern California) Region\nNeeds location constraint us-west-1.", + }, { + Value: "ca-central-1", + Help: "Canada (Central) Region\nNeeds location constraint ca-central-1.", + }, { + Value: "eu-west-1", + Help: "EU (Ireland) Region\nNeeds location constraint EU or eu-west-1.", + }, { + Value: "eu-west-2", + Help: "EU (London) Region\nNeeds location constraint eu-west-2.", + }, { + Value: "eu-central-1", + Help: "EU (Frankfurt) Region\nNeeds location constraint eu-central-1.", + }, { + Value: "ap-southeast-1", + Help: "Asia Pacific (Singapore) Region\nNeeds location constraint ap-southeast-1.", + }, { + Value: "ap-southeast-2", + Help: "Asia Pacific (Sydney) Region\nNeeds location constraint ap-southeast-2.", + }, { + Value: "ap-northeast-1", + Help: "Asia Pacific (Tokyo) Region\nNeeds location constraint ap-northeast-1.", + }, { + Value: "ap-northeast-2", + Help: "Asia Pacific (Seoul)\nNeeds location constraint ap-northeast-2.", + }, { + Value: "ap-south-1", + Help: "Asia Pacific (Mumbai)\nNeeds location constraint ap-south-1.", + }, { + Value: "sa-east-1", + Help: "South America (Sao Paulo) Region\nNeeds location constraint sa-east-1.", + }, { + Value: "other-v2-signature", + Help: "Use this only if v4 signatures don't work, eg pre Jewel/v10 CEPH.\nSet this and make sure you set the endpoint.", + }}, }, { - Value: "ca-central-1", - Help: "Canada (Central) Region\nNeeds location constraint ca-central-1.", + Name: "endpoint", + Help: "Endpoint for S3 API.\nLeave blank if using AWS to use the default endpoint for the region.\nSpecify if using an S3 clone such as Ceph.", + Provider: "AWS,Ceph,Dreamhost,Minio,Other", }, { - Value: "eu-west-1", - Help: "EU (Ireland) Region\nNeeds location constraint EU or eu-west-1.", + Name: "endpoint", + Help: "Endpoint for IBM COS S3 API.\nSpecify if using an IBM COS On Premise.", + Provider: "IBMCOS", + Examples: []fs.OptionExample{{ + Value: "s3-api.us-geo.objectstorage.softlayer.net", + Help: "US Cross Region Endpoint", + }, { + Value: "s3-api.dal.us-geo.objectstorage.softlayer.net", + Help: "US Cross Region Dallas Endpoint", + }, { + Value: "s3-api.wdc-us-geo.objectstorage.softlayer.net", + Help: "US Cross Region Washington DC Endpoint", + }, { + Value: "s3-api.sjc-us-geo.objectstorage.softlayer.net", + Help: "US Cross Region San Jose Endpoint", + }, { + Value: "s3-api.us-geo.objectstorage.service.networklayer.com", + Help: "US Cross Region Private Endpoint", + }, { + Value: "s3-api.dal-us-geo.objectstorage.service.networklayer.com", + Help: "US Cross Region Dallas Private Endpoint", + }, { + Value: "s3-api.wdc-us-geo.objectstorage.service.networklayer.com", + Help: "US Cross Region Washington DC Private Endpoint", + }, { + Value: "s3-api.sjc-us-geo.objectstorage.service.networklayer.com", + Help: "US Cross Region San Jose Private Endpoint", + }, { + Value: "s3.us-east.objectstorage.softlayer.net", + Help: "US Region East Endpoint", + }, { + Value: "s3.us-east.objectstorage.service.networklayer.com", + Help: "US Region East Private Endpoint", + }, { + Value: "s3.us-south.objectstorage.softlayer.net", + Help: "US Region South Endpoint", + }, { + Value: "s3.us-south.objectstorage.service.networklayer.com", + Help: "US Region South Private Endpoint", + }, { + Value: "s3.eu-geo.objectstorage.softlayer.net", + Help: "EU Cross Region Endpoint", + }, { + Value: "s3.fra-eu-geo.objectstorage.softlayer.net", + Help: "EU Cross Region Frankfurt Endpoint", + }, { + Value: "s3.mil-eu-geo.objectstorage.softlayer.net", + Help: "EU Cross Region Milan Endpoint", + }, { + Value: "s3.ams-eu-geo.objectstorage.softlayer.net", + Help: "EU Cross Region Amsterdam Endpoint", + }, { + Value: "s3.eu-geo.objectstorage.service.networklayer.com", + Help: "EU Cross Region Private Endpoint", + }, { + Value: "s3.fra-eu-geo.objectstorage.service.networklayer.com", + Help: "EU Cross Region Frankfurt Private Endpoint", + }, { + Value: "s3.mil-eu-geo.objectstorage.service.networklayer.com", + Help: "EU Cross Region Milan Private Endpoint", + }, { + Value: "s3.ams-eu-geo.objectstorage.service.networklayer.com", + Help: "EU Cross Region Amsterdam Private Endpoint", + }, { + Value: "s3.eu-gb.objectstorage.softlayer.net", + Help: "Great Britan Endpoint", + }, { + Value: "s3.eu-gb.objectstorage.service.networklayer.com", + Help: "Great Britan Private Endpoint", + }, { + Value: "s3.ap-geo.objectstorage.softlayer.net", + Help: "APAC Cross Regional Endpoint", + }, { + Value: "s3.tok-ap-geo.objectstorage.softlayer.net", + Help: "APAC Cross Regional Tokyo Endpoint", + }, { + Value: "s3.hkg-ap-geo.objectstorage.softlayer.net", + Help: "APAC Cross Regional HongKong Endpoint", + }, { + Value: "s3.seo-ap-geo.objectstorage.softlayer.net", + Help: "APAC Cross Regional Seoul Endpoint", + }, { + Value: "s3.ap-geo.objectstorage.service.networklayer.com", + Help: "APAC Cross Regional Private Endpoint", + }, { + Value: "s3.tok-ap-geo.objectstorage.service.networklayer.com", + Help: "APAC Cross Regional Tokyo Private Endpoint", + }, { + Value: "s3.hkg-ap-geo.objectstorage.service.networklayer.com", + Help: "APAC Cross Regional HongKong Private Endpoint", + }, { + Value: "s3.seo-ap-geo.objectstorage.service.networklayer.com", + Help: "APAC Cross Regional Seoul Private Endpoint", + }, { + Value: "s3.mel01.objectstorage.softlayer.net", + Help: "Melbourne Single Site Endpoint", + }, { + Value: "s3.mel01.objectstorage.service.networklayer.com", + Help: "Melbourne Single Site Private Endpoint", + }, { + Value: "s3.tor01.objectstorage.softlayer.net", + Help: "Toronto Single Site Endpoint", + }, { + Value: "s3.tor01.objectstorage.service.networklayer.com", + Help: "Toronto Single Site Private Endpoint", + }}, }, { - Value: "eu-west-2", - Help: "EU (London) Region\nNeeds location constraint eu-west-2.", + Name: "location_constraint", + Help: "Location constraint - must be set to match the Region. Used when creating buckets only.", + Provider: "AWS,Ceph,Dreamhost,Minio", + Examples: []fs.OptionExample{{ + Value: "", + Help: "Empty for US Region, Northern Virginia or Pacific Northwest.", + Provider: "AWS,Ceph,Dreamhost,Minio,Other", + }, { + Value: "us-east-2", + Help: "US East (Ohio) Region.", + }, { + Value: "us-west-2", + Help: "US West (Oregon) Region.", + }, { + Value: "us-west-1", + Help: "US West (Northern California) Region.", + }, { + Value: "ca-central-1", + Help: "Canada (Central) Region.", + }, { + Value: "eu-west-1", + Help: "EU (Ireland) Region.", + }, { + Value: "eu-west-2", + Help: "EU (London) Region.", + }, { + Value: "EU", + Help: "EU Region.", + }, { + Value: "ap-southeast-1", + Help: "Asia Pacific (Singapore) Region.", + }, { + Value: "ap-southeast-2", + Help: "Asia Pacific (Sydney) Region.", + }, { + Value: "ap-northeast-1", + Help: "Asia Pacific (Tokyo) Region.", + }, { + Value: "ap-northeast-2", + Help: "Asia Pacific (Seoul)", + }, { + Value: "ap-south-1", + Help: "Asia Pacific (Mumbai)", + }, { + Value: "sa-east-1", + Help: "South America (Sao Paulo) Region.", + }}, }, { - Value: "eu-central-1", - Help: "EU (Frankfurt) Region\nNeeds location constraint eu-central-1.", + Name: "location_constraint", + Help: "Location constraint - must match endpoint when using IBM Cloud Public. For on-prem COS, do not make a selection from this list, hit enter", + Provider: "IBMCOS", + Examples: []fs.OptionExample{{ + Value: "us-standard", + Help: "US Cross Region Standard", + }, { + Value: "us-vault", + Help: "US Cross Region Vault", + }, { + Value: "us-cold", + Help: "US Cross Region Cold", + }, { + Value: "us-flex", + Help: "US Cross Region Flex", + }, { + Value: "us-east-standard", + Help: "US East Region Standard", + }, { + Value: "us-east-vault", + Help: "US East Region Vault", + }, { + Value: "us-east-cold", + Help: "US East Region Cold", + }, { + Value: "us-east-flex", + Help: "US East Region Flex", + }, { + Value: "us-south-standard", + Help: "US Sout hRegion Standard", + }, { + Value: "us-south-vault", + Help: "US South Region Vault", + }, { + Value: "us-south-cold", + Help: "US South Region Cold", + }, { + Value: "us-south-flex", + Help: "US South Region Flex", + }, { + Value: "eu-standard", + Help: "EU Cross Region Standard", + }, { + Value: "eu-vault", + Help: "EU Cross Region Vault", + }, { + Value: "eu-cold", + Help: "EU Cross Region Cold", + }, { + Value: "eu-flex", + Help: "EU Cross Region Flex", + }, { + Value: "eu-gb-standard", + Help: "Great Britan Standard", + }, { + Value: "eu-gb-vault", + Help: "Great Britan Vault", + }, { + Value: "eu-gb-cold", + Help: "Great Britan Cold", + }, { + Value: "eu-gb-flex", + Help: "Great Britan Flex", + }, { + Value: "ap-standard", + Help: "APAC Standard", + }, { + Value: "ap-vault", + Help: "APAC Vault", + }, { + Value: "ap-cold", + Help: "APAC Cold", + }, { + Value: "ap-flex", + Help: "APAC Flex", + }, { + Value: "mel01-standard", + Help: "Melbourne Standard", + }, { + Value: "mel01-vault", + Help: "Melbourne Vault", + }, { + Value: "mel01-cold", + Help: "Melbourne Cold", + }, { + Value: "mel01-flex", + Help: "Melbourne Flex", + }, { + Value: "tor01-standard", + Help: "Toronto Standard", + }, { + Value: "tor01-vault", + Help: "Toronto Vault", + }, { + Value: "tor01-cold", + Help: "Toronto Cold", + }, { + Value: "tor01-flex", + Help: "Toronto Flex", + }}, }, { - Value: "ap-southeast-1", - Help: "Asia Pacific (Singapore) Region\nNeeds location constraint ap-southeast-1.", + Name: "acl", + Help: "Canned ACL used when creating buckets and/or storing objects in S3.\nFor more info visit https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl", + Provider: "AWS,Ceph,Dreamhost,IBMCOS,Minio,Other", + Examples: []fs.OptionExample{{ + Value: "private", + Help: "Owner gets FULL_CONTROL. No one else has access rights (default).", + Provider: "AWS,Ceph,Dreamhost,Minio,Other", + }, { + Value: "public-read", + Help: "Owner gets FULL_CONTROL. The AllUsers group gets READ access.", + Provider: "AWS,Ceph,Dreamhost,Minio,Other", + }, { + Value: "public-read-write", + Help: "Owner gets FULL_CONTROL. The AllUsers group gets READ and WRITE access.\nGranting this on a bucket is generally not recommended.", + Provider: "AWS,Ceph,Dreamhost,Minio,Other", + }, { + Value: "authenticated-read", + Help: "Owner gets FULL_CONTROL. The AuthenticatedUsers group gets READ access.", + Provider: "AWS,Ceph,Dreamhost,Minio,Other", + }, { + Value: "bucket-owner-read", + Help: "Object owner gets FULL_CONTROL. Bucket owner gets READ access.\nIf you specify this canned ACL when creating a bucket, Amazon S3 ignores it.", + Provider: "AWS,Ceph,Dreamhost,Minio,Other", + }, { + Value: "bucket-owner-full-control", + Help: "Both the object owner and the bucket owner get FULL_CONTROL over the object.\nIf you specify this canned ACL when creating a bucket, Amazon S3 ignores it.", + Provider: "AWS,Ceph,Dreamhost,Minio,Other", + }, { + Value: "private", + Help: "Owner gets FULL_CONTROL. No one else has access rights (default). This acl is available on IBM Cloud (Infra), IBM Cloud (Storage), On-Premise COS", + Provider: "IBMCOS", + }, { + Value: "public-read", + Help: "Owner gets FULL_CONTROL. The AllUsers group gets READ access. This acl is available on IBM Cloud (Infra), IBM Cloud (Storage), On-Premise IBM COS", + Provider: "IBMCOS", + }, { + Value: "public-read-write", + Help: "Owner gets FULL_CONTROL. The AllUsers group gets READ and WRITE access. This acl is available on IBM Cloud (Infra), On-Premise IBM COS", + Provider: "IBMCOS", + }, { + Value: "authenticated-read", + Help: "Owner gets FULL_CONTROL. The AuthenticatedUsers group gets READ access. Not supported on Buckets. This acl is available on IBM Cloud (Infra) and On-Premise IBM COS", + Provider: "IBMCOS", + }}, }, { - Value: "ap-southeast-2", - Help: "Asia Pacific (Sydney) Region\nNeeds location constraint ap-southeast-2.", + Name: "server_side_encryption", + Help: "The server-side encryption algorithm used when storing this object in S3.", + Provider: "AWS,Ceph,Dreamhost,Minio,Other", + Examples: []fs.OptionExample{{ + Value: "", + Help: "None", + }, { + Value: "AES256", + Help: "AES256", + }}, }, { - Value: "ap-northeast-1", - Help: "Asia Pacific (Tokyo) Region\nNeeds location constraint ap-northeast-1.", - }, { - Value: "ap-northeast-2", - Help: "Asia Pacific (Seoul)\nNeeds location constraint ap-northeast-2.", - }, { - Value: "ap-south-1", - Help: "Asia Pacific (Mumbai)\nNeeds location constraint ap-south-1.", - }, { - Value: "sa-east-1", - Help: "South America (Sao Paulo) Region\nNeeds location constraint sa-east-1.", - }, { - Value: "other-v2-signature", - Help: "Use this only if v4 signatures don't work, eg pre Jewel/v10 CEPH.\nSet this and make sure you set the endpoint.", - }}, - }, { - Name: "endpoint", - Help: "Endpoint for S3 API.\nLeave blank if using AWS to use the default endpoint for the region.\nSpecify if using an S3 clone such as Ceph.", - }, { - Name: "location_constraint", - Help: "Location constraint - must be set to match the Region. Used when creating buckets only.", - Examples: []fs.OptionExample{{ - Value: "", - Help: "Empty for US Region, Northern Virginia or Pacific Northwest.", - }, { - Value: "us-east-2", - Help: "US East (Ohio) Region.", - }, { - Value: "us-west-2", - Help: "US West (Oregon) Region.", - }, { - Value: "us-west-1", - Help: "US West (Northern California) Region.", - }, { - Value: "ca-central-1", - Help: "Canada (Central) Region.", - }, { - Value: "eu-west-1", - Help: "EU (Ireland) Region.", - }, { - Value: "eu-west-2", - Help: "EU (London) Region.", - }, { - Value: "EU", - Help: "EU Region.", - }, { - Value: "ap-southeast-1", - Help: "Asia Pacific (Singapore) Region.", - }, { - Value: "ap-southeast-2", - Help: "Asia Pacific (Sydney) Region.", - }, { - Value: "ap-northeast-1", - Help: "Asia Pacific (Tokyo) Region.", - }, { - Value: "ap-northeast-2", - Help: "Asia Pacific (Seoul)", - }, { - Value: "ap-south-1", - Help: "Asia Pacific (Mumbai)", - }, { - Value: "sa-east-1", - Help: "South America (Sao Paulo) Region.", - }}, - }, { - Name: "acl", - Help: "Canned ACL used when creating buckets and/or storing objects in S3.\nFor more info visit https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl", - Examples: []fs.OptionExample{{ - Value: "private", - Help: "Owner gets FULL_CONTROL. No one else has access rights (default).", - }, { - Value: "public-read", - Help: "Owner gets FULL_CONTROL. The AllUsers group gets READ access.", - }, { - Value: "public-read-write", - Help: "Owner gets FULL_CONTROL. The AllUsers group gets READ and WRITE access.\nGranting this on a bucket is generally not recommended.", - }, { - Value: "authenticated-read", - Help: "Owner gets FULL_CONTROL. The AuthenticatedUsers group gets READ access.", - }, { - Value: "bucket-owner-read", - Help: "Object owner gets FULL_CONTROL. Bucket owner gets READ access.\nIf you specify this canned ACL when creating a bucket, Amazon S3 ignores it.", - }, { - Value: "bucket-owner-full-control", - Help: "Both the object owner and the bucket owner get FULL_CONTROL over the object.\nIf you specify this canned ACL when creating a bucket, Amazon S3 ignores it.", - }}, - }, { - Name: "server_side_encryption", - Help: "The server-side encryption algorithm used when storing this object in S3.", - Examples: []fs.OptionExample{{ - Value: "", - Help: "None", - }, { - Value: "AES256", - Help: "AES256", - }}, - }, { - Name: "storage_class", - Help: "The storage class to use when storing objects in S3.", - Examples: []fs.OptionExample{{ - Value: "", - Help: "Default", - }, { - Value: "STANDARD", - Help: "Standard storage class", - }, { - Value: "REDUCED_REDUNDANCY", - Help: "Reduced redundancy storage class", - }, { - Value: "STANDARD_IA", - Help: "Standard Infrequent Access storage class", - }, { - Value: "ONEZONE_IA", - Help: "One Zone Infrequent Access storage class", - }}, - }}, + Name: "storage_class", + Help: "The storage class to use when storing objects in S3.", + Provider: "AWS,Ceph,Dreamhost,Minio,Other", + Examples: []fs.OptionExample{{ + Value: "", + Help: "Default", + }, { + Value: "STANDARD", + Help: "Standard storage class", + }, { + Value: "REDUCED_REDUNDANCY", + Help: "Reduced redundancy storage class", + }, { + Value: "STANDARD_IA", + Help: "Standard Infrequent Access storage class", + }, { + Value: "ONEZONE_IA", + Help: "One Zone Infrequent Access storage class", + }}, + }, + }, }) flags.VarP(&s3ChunkSize, "s3-chunk-size", "", "Chunk size to use for uploading") } @@ -1052,6 +1322,7 @@ func (o *Object) Update(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOptio u.LeavePartsOnError = false u.S3 = o.fs.c u.PartSize = int64(s3ChunkSize) + u.PartSize = s3manager.MinUploadPartSize if size == -1 { // Make parts as small as possible while still being able to upload to the diff --git a/docs/content/about.md b/docs/content/about.md index d71156f8c..cd697316b 100644 --- a/docs/content/about.md +++ b/docs/content/about.md @@ -26,7 +26,7 @@ Rclone is a command line program to sync files and directories to and from: * {{< provider name="Google Drive" home="https://www.google.com/drive/" config="/drive/" >}} * {{< provider name="HTTP" home="https://en.wikipedia.org/wiki/Hypertext_Transfer_Protocol" config="/http/" >}} * {{< provider name="Hubic" home="https://hubic.com/" config="/hubic/" >}} -* {{< provider name="IBM COS S3" home="http://www.ibm.com/cloud/object-storage" config="/s3/" >}} +* {{< provider name="IBM COS S3" home="http://www.ibm.com/cloud/object-storage" config="/s3/#ibm-cos-s3" >}} * {{< provider name="Memset Memstore" home="https://www.memset.com/cloud/storage/" config="/swift/" >}} * {{< provider name="Microsoft Azure Blob Storage" home="https://azure.microsoft.com/en-us/services/storage/blobs/" config="/azureblob/" >}} * {{< provider name="Microsoft OneDrive" home="https://onedrive.live.com/" config="/onedrive/" >}} diff --git a/docs/content/s3.md b/docs/content/s3.md index 33ec590e6..96d0a5898 100644 --- a/docs/content/s3.md +++ b/docs/content/s3.md @@ -4,8 +4,17 @@ description: "Rclone docs for Amazon S3" date: "2016-07-11" --- - Amazon S3 ---------------------------------------- + Amazon S3 Storage Providers +-------------------------------------------------------- +* {{< provider name="AWS S3" home="https://aws.amazon.com/s3/" config="/s3/" >}} +* {{< provider name="Ceph" home="http://ceph.com/" config="/s3/#ceph" >}} +* {{< provider name="DigitalOcean Spaces" home="https://www.digitalocean.com/products/object-storage/" config="/s3/#digitalocean-spaces" >}} +* {{< provider name="IBM COS S3" home="http://www.ibm.com/cloud/object-storage" config="/s3/#ibm-cos-s3" >}} +* {{< provider name="Minio" home="https://www.minio.io/" config="/s3/#minio" >}} +* {{< provider name="Wasabi" home="https://wasabi.com/" config="/s3/#wasabi" >}} + + +## AWS S3 {#amazon-s3} Paths are specified as `remote:bucket` (or `remote:` for the `lsd` command.) You may put subdirectories in too, eg `remote:bucket/path/to/dir`. @@ -29,14 +38,27 @@ Choose a number from below, or type in your own value \ "alias" 2 / Amazon Drive \ "amazon cloud drive" - 3 / Amazon S3 (also Dreamhost, Ceph, Minio) + 3 / Amazon S3 Complaint Storage Providers (Dreamhost, Ceph, Minio, IBM COS) \ "s3" 4 / Backblaze B2 \ "b2" [snip] 23 / http Connection \ "http" -Storage> s3 +Storage> 3 +Choose the S3 provider. +Choose a number from below, or type in your own value + 1 / Choose this option to configure Storage to AWS S3 + \ "AWS" + 2 / Choose this option to configure Storage to Ceph Systems + \ "Ceph" + 3 / Choose this option to configure Storage to Dreamhost + \ "Dreamhost" + 4 / Choose this option to the configure Storage to IBM COS S3 + \ "IBMCOS" + 5 / Choose this option to the configure Storage to Minio + \ "Minio" +Provider>1 Get AWS credentials from runtime (environment variables or EC2/ECS meta data if no env vars). Only applies if access_key_id and secret_access_key is blank. Choose a number from below, or type in your own value 1 / Enter AWS credentials in the next step @@ -100,7 +122,7 @@ region> 1 Endpoint for S3 API. Leave blank if using AWS to use the default endpoint for the region. Specify if using an S3 clone such as Ceph. -endpoint> +endpoint> Location constraint - must be set to match the Region. Used when creating buckets only. Choose a number from below, or type in your own value 1 / Empty for US Region, Northern Virginia or Pacific Northwest. @@ -178,11 +200,11 @@ env_auth = false access_key_id = XXX secret_access_key = YYY region = us-east-1 -endpoint = -location_constraint = +endpoint = +location_constraint = acl = private -server_side_encryption = -storage_class = +server_side_encryption = +storage_class = -------------------- y) Yes this is OK e) Edit this remote @@ -268,7 +290,7 @@ credentials then S3 interaction will be non-authenticated (see below). ### S3 Permissions ### -When using the `sync` subcommand of `rclone` the following minimum +When using the `sync` subcommand of `rclone` the following minimum permissions are required to be available on the bucket being written to: * `ListBucket` @@ -308,10 +330,10 @@ Notes on above: 1. This is a policy that can be used when creating bucket. It assumes that `USER_NAME` has been created. -2. The Resource entry must include both resource ARNs, as one implies +2. The Resource entry must include both resource ARNs, as one implies the bucket and the other implies the bucket's objects. -For reference, [here's an Ansible script](https://gist.github.com/ebridges/ebfc9042dd7c756cd101cfa807b7ae2b) +For reference, [here's an Ansible script](https://gist.github.com/ebridges/ebfc9042dd7c756cd101cfa807b7ae2b) that will generate one or more buckets that will work with `rclone sync`. ### Key Management System (KMS) ### @@ -365,6 +387,7 @@ Note that 2 chunks of this size are buffered in memory per transfer. If you are transferring large files over high speed links and you have enough memory, then increasing this will speed up the transfers. + ### Anonymous access to public buckets ### If you want to use rclone to access a public bucket, configure with a @@ -427,12 +450,12 @@ type = s3 env_auth = false access_key_id = XXX secret_access_key = YYY -region = +region = endpoint = https://ceph.endpoint.example.com -location_constraint = -acl = -server_side_encryption = -storage_class = +location_constraint = +acl = +server_side_encryption = +storage_class = ``` Note also that Ceph sometimes puts `/` in the passwords it gives @@ -498,11 +521,11 @@ Storage> s3 env_auth> 1 access_key_id> YOUR_ACCESS_KEY secret_access_key> YOUR_SECRET_KEY -region> +region> endpoint> nyc3.digitaloceanspaces.com -location_constraint> -acl> -storage_class> +location_constraint> +acl> +storage_class> ``` The resulting configuration file should look like: @@ -513,12 +536,12 @@ type = s3 env_auth = false access_key_id = YOUR_ACCESS_KEY secret_access_key = YOUR_SECRET_KEY -region = +region = endpoint = nyc3.digitaloceanspaces.com -location_constraint = -acl = -server_side_encryption = -storage_class = +location_constraint = +acl = +server_side_encryption = +storage_class = ``` Once configured, you can create a new Space and begin copying files. For example: @@ -545,30 +568,41 @@ To configure access to IBM COS S3, follow the steps below: 2. Enter the name for the configuration ``` - name> IBM-COS-XREGION + name> ``` 3. Select "s3" storage. ``` - Type of storage to configure. - Choose a number from below, or type in your own value - 1 / Amazon Drive +Choose a number from below, or type in your own value + 1 / Alias for a existing remote + \ "alias" + 2 / Amazon Drive \ "amazon cloud drive" - 2 / Amazon S3 (also Dreamhost, Ceph, Minio, IBM COS(S3)) + 3 / Amazon S3 Complaint Storage Providers (Dreamhost, Ceph, Minio, IBM COS) \ "s3" - 3 / Backblaze B2 - Storage> 2 + 4 / Backblaze B2 + \ "b2" +[snip] + 23 / http Connection + \ "http" +Storage> 3 ``` -4. Select "Enter AWS credentials…" +4. Select IBM COS as the S3 Storage Provider. ``` - Get AWS credentials from runtime (environment variables or EC2/ECS meta data if no env vars). Only applies if access_key_id and secret_access_key is blank. - Choose a number from below, or type in your own value - 1 / Enter AWS credentials in the next step - \ "false" - 2 / Get AWS credentials from the environment (env vars or IAM) - \ "true" - env_auth> 1 +Choose the S3 provider. +Choose a number from below, or type in your own value + 1 / Choose this option to configure Storage to AWS S3 + \ "AWS" + 2 / Choose this option to configure Storage to Ceph Systems + \ "Ceph" + 3 / Choose this option to configure Storage to Dreamhost + \ "Dreamhost" + 4 / Choose this option to the configure Storage to IBM COS S3 + \ "IBMCOS" + 5 / Choose this option to the configure Storage to Minio + \ "Minio" + Provider>4 ``` 5. Enter the Access Key and Secret. @@ -579,111 +613,94 @@ To configure access to IBM COS S3, follow the steps below: secret_access_key> <> ``` -6. Select "other-v4-signature" region. +6. Specify the endpoint for IBM COS. For Public IBM COS, choose from the option below. For On Premise IBM COS, enter an enpoint address. ``` - Region to connect to. + Endpoint for IBM COS S3 API. + Specify if using an IBM COS On Premise. Choose a number from below, or type in your own value - / The default endpoint - a good choice if you are unsure. - 1 | US Region, Northern Virginia or Pacific Northwest. - | Leave location constraint empty. - \ "us-east-1" - / US East (Ohio) Region - 2 | Needs location constraint us-east-2. - \ "us-east-2" - / US West (Oregon) Region - …… - 15 | eg Ceph/Dreamhost - | set this and make sure you set the endpoint. - \ "other-v2-signature" - / If using an S3 clone that understands v4 signatures set this - 16 | and make sure you set the endpoint. - \ "other-v4-signature - region> 16 + 1 / US Cross Region Endpoint + \ "s3-api.us-geo.objectstorage.softlayer.net" + 2 / US Cross Region Dallas Endpoint + \ "s3-api.dal.us-geo.objectstorage.softlayer.net" + 3 / US Cross Region Washington DC Endpoint + \ "s3-api.wdc-us-geo.objectstorage.softlayer.net" + 4 / US Cross Region San Jose Endpoint + \ "s3-api.sjc-us-geo.objectstorage.softlayer.net" + 5 / US Cross Region Private Endpoint + \ "s3-api.us-geo.objectstorage.service.networklayer.com" + 6 / US Cross Region Dallas Private Endpoint + \ "s3-api.dal-us-geo.objectstorage.service.networklayer.com" + 7 / US Cross Region Washington DC Private Endpoint + \ "s3-api.wdc-us-geo.objectstorage.service.networklayer.com" + 8 / US Cross Region San Jose Private Endpoint + \ "s3-api.sjc-us-geo.objectstorage.service.networklayer.com" + 9 / US Region East Endpoint + \ "s3.us-east.objectstorage.softlayer.net" + 10 / US Region East Private Endpoint + \ "s3.us-east.objectstorage.service.networklayer.com" + 11 / US Region South Endpoint +[snip] + 34 / Toronto Single Site Private Endpoint + \ "s3.tor01.objectstorage.service.networklayer.com" + endpoint>1 ``` -7. Enter the endpoint FQDN. + +7. Specify a IBM COS Location Constraint. The location constraint must match endpoint when using IBM Cloud Public. For on-prem COS, do not make a selection from this list, hit enter ``` - Leave blank if using AWS to use the default endpoint for the region. - Specify if using an S3 clone such as Ceph. - endpoint> s3-api.us-geo.objectstorage.softlayer.net + 1 / US Cross Region Standard + \ "us-standard" + 2 / US Cross Region Vault + \ "us-vault" + 3 / US Cross Region Cold + \ "us-cold" + 4 / US Cross Region Flex + \ "us-flex" + 5 / US East Region Standard + \ "us-east-standard" + 6 / US East Region Vault + \ "us-east-vault" + 7 / US East Region Cold + \ "us-east-cold" + 8 / US East Region Flex + \ "us-east-flex" + 9 / US South Region Standard + \ "us-south-standard" + 10 / US South Region Vault + \ "us-south-vault" +[snip] + 32 / Toronto Flex + \ "tor01-flex" +location_constraint>1 ``` -8. Specify a IBM COS Location Constraint. - a. Currently, the only IBM COS values for LocationConstraint are: - us-standard / us-vault / us-cold / us-flex - us-east-standard / us-east-vault / us-east-cold / us-east-flex - us-south-standard / us-south-vault / us-south-cold / us-south-flex - eu-standard / eu-vault / eu-cold / eu-flex +9. Specify a canned ACL. IBM Cloud (Strorage) supports "public-read" and "private". IBM Cloud(Infra) supports all the canned ACLs. On-Premise COS supports all the canned ACLs. ``` - Location constraint - must be set to match the Region. Used when creating buckets only. - Choose a number from below, or type in your own value - 1 / Empty for US Region, Northern Virginia or Pacific Northwest. - \ "" - 2 / US East (Ohio) Region. - \ "us-east-2" - …… - location_constraint> us-standard +Canned ACL used when creating buckets and/or storing objects in S3. +For more info visit https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl +Choose a number from below, or type in your own value + 1 / Owner gets FULL_CONTROL. No one else has access rights (default). This acl is available on IBM Cloud (Infra), IBM Cloud (Storage), On-Premise COS + \ "private" + 2 / Owner gets FULL_CONTROL. The AllUsers group gets READ access. This acl is available on IBM Cloud (Infra), IBM Cloud (Storage), On-Premise IBM COS + \ "public-read" + 3 / Owner gets FULL_CONTROL. The AllUsers group gets READ and WRITE access. This acl is available on IBM Cloud (Infra), On-Premise IBM COS + \ "public-read-write" + 4 / Owner gets FULL_CONTROL. The AuthenticatedUsers group gets READ access. Not supported on Buckets. This acl is available on IBM Cloud (Infra) and On-Premise IBM COS + \ "authenticated-read" +acl> 1 ``` -9. Specify a canned ACL. IBM COS on Bluemix(IBM Cloud) supports "public-read" and "private". IBM COS Infrastrucure on Bluemix(IBM Cloud) supports all the canned ACLs. On-Prem COS supports all the canned ACLs. -``` - Canned ACL used when creating buckets and/or storing objects in S3. - For more info visit https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl - Choose a number from below, or type in your own value - 1 / Owner gets FULL_CONTROL. No one else has access rights (default). - \ "private" - 2 / Owner gets FULL_CONTROL. The AllUsers group gets READ access. - \ "public-read" - / Owner gets FULL_CONTROL. The AllUsers group gets READ and WRITE access. - 3 | Granting this on a bucket is generally not recommended. - \ "public-read-write" - 4 / Owner gets FULL_CONTROL. The AuthenticatedUsers group gets READ access. - \ "authenticated-read" - / Object owner gets FULL_CONTROL. Bucket owner gets READ access. - 5 | If you specify this canned ACL when creating a bucket, Amazon S3 ignores it. - \ "bucket-owner-read" - / Both the object owner and the bucket owner get FULL_CONTROL over the object. - 6 | If you specify this canned ACL when creating a bucket, Amazon S3 ignores it. - \ "bucket-owner-full-control" - acl> 1 -``` - -10. Set the SSE option to "None". -``` - Choose a number from below, or type in your own value - 1 / None - \ "" - 2 / AES256 - \ "AES256" - server_side_encryption> 1 -``` - -11. Set the storage class to "None" (IBM COS uses the LocationConstraint at the bucket level). -``` - The storage class to use when storing objects in S3. - Choose a number from below, or type in your own value - 1 / Default - \ "" - 2 / Standard storage class - \ "STANDARD" - 3 / Reduced redundancy storage class - \ "REDUCED_REDUNDANCY" - 4 / Standard Infrequent Access storage class - \ "STANDARD_IA" - storage_class> -``` 12. Review the displayed configuration and accept to save the "remote" then quit. The config file should look like this ``` - env_auth = false - access_key_id = <> - secret_access_key = <> - region = other-v4-signature + [xxx] + type = s3 + Provider = IBMCOS + access_key_id = xxx + secret_access_key = yyy endpoint = s3-api.us-geo.objectstorage.softlayer.net location_constraint = us-standard acl = private - server_side_encryption = - storage_class = ``` 13. Execute rclone commands @@ -822,21 +839,21 @@ Choose a number from below, or type in your own value 1 / Empty for US Region, Northern Virginia or Pacific Northwest. \ "" [snip] -location_constraint> +location_constraint> Canned ACL used when creating buckets and/or storing objects in S3. For more info visit https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl Choose a number from below, or type in your own value 1 / Owner gets FULL_CONTROL. No one else has access rights (default). \ "private" [snip] -acl> +acl> The server-side encryption algorithm used when storing this object in S3. Choose a number from below, or type in your own value 1 / None \ "" 2 / AES256 \ "AES256" -server_side_encryption> +server_side_encryption> The storage class to use when storing objects in S3. Choose a number from below, or type in your own value 1 / Default @@ -847,7 +864,7 @@ Choose a number from below, or type in your own value \ "REDUCED_REDUNDANCY" 4 / Standard Infrequent Access storage class \ "STANDARD_IA" -storage_class> +storage_class> Remote config -------------------- [wasabi] @@ -856,10 +873,10 @@ access_key_id = YOURACCESSKEY secret_access_key = YOURSECRETACCESSKEY region = us-east-1 endpoint = s3.wasabisys.com -location_constraint = -acl = -server_side_encryption = -storage_class = +location_constraint = +acl = +server_side_encryption = +storage_class = -------------------- y) Yes this is OK e) Edit this remote @@ -876,8 +893,8 @@ access_key_id = YOURACCESSKEY secret_access_key = YOURSECRETACCESSKEY region = us-east-1 endpoint = s3.wasabisys.com -location_constraint = -acl = -server_side_encryption = -storage_class = +location_constraint = +acl = +server_side_encryption = +storage_class = ``` diff --git a/fs/config/config.go b/fs/config/config.go index 6aabe5094..3e295f81a 100644 --- a/fs/config/config.go +++ b/fs/config/config.go @@ -684,7 +684,8 @@ func RemoteConfig(name string) { } // ChooseOption asks the user to choose an option -func ChooseOption(o *fs.Option) string { +func ChooseOption(o *fs.Option, name string) string { + var subProvider = getConfigData().MustValue(name, "Provider", "") fmt.Println(o.Help) if o.IsPassword { actions := []string{"yYes type in my own password", "gGenerate random password"} @@ -726,8 +727,17 @@ func ChooseOption(o *fs.Option) string { var values []string var help []string for _, example := range o.Examples { - values = append(values, example.Value) - help = append(help, example.Help) + if example.Provider != "" { + if strings.Contains(example.Provider, subProvider) { + values = append(values, example.Value) + help = append(help, example.Help) + } else { + continue + } + } else { + values = append(values, example.Value) + help = append(help, example.Help) + } } return Choose(o.Name, values, help, true) } @@ -836,11 +846,18 @@ func NewRemoteName() (name string) { // NewRemote make a new remote from its name func NewRemote(name string) { - newType := ChooseOption(fsOption()) + newType := ChooseOption(fsOption(), name) getConfigData().SetValue(name, "type", newType) fs := fs.MustFind(newType) for _, option := range fs.Options { - getConfigData().SetValue(name, option.Name, ChooseOption(&option)) + var subProvider = getConfigData().MustValue(name, "Provider", "") + if option.Provider != "" { + if strings.Contains(option.Provider, subProvider) { + getConfigData().SetValue(name, option.Name, ChooseOption(&option, name)) + } + } else { + getConfigData().SetValue(name, option.Name, ChooseOption(&option, name)) + } } RemoteConfig(name) if OkRemote(name) { @@ -854,14 +871,24 @@ func NewRemote(name string) { func EditRemote(fs *fs.RegInfo, name string) { ShowRemote(name) fmt.Printf("Edit remote\n") + var subProvider = "" for { for _, option := range fs.Options { key := option.Name value := FileGet(name, key) + if strings.Compare(key, "Provider") == 0 { + subProvider = value + continue + } + if option.Provider != "" { + if !(strings.Contains(option.Provider, subProvider)) { + continue + } + } fmt.Printf("Value %q = %q\n", key, value) fmt.Printf("Edit? (y/n)>\n") if Confirm() { - newValue := ChooseOption(&option) + newValue := ChooseOption(&option, name) getConfigData().SetValue(name, key, newValue) } } diff --git a/fs/fs.go b/fs/fs.go index f60e507fc..e080f922a 100644 --- a/fs/fs.go +++ b/fs/fs.go @@ -83,6 +83,7 @@ type RegInfo struct { type Option struct { Name string Help string + Provider string Optional bool IsPassword bool Examples OptionExamples `json:",omitempty"` @@ -105,8 +106,9 @@ func (os OptionExamples) Sort() { sort.Sort(os) } // OptionExample describes an example for an Option type OptionExample struct { - Value string - Help string + Value string + Help string + Provider string } // Register a filesystem