2015-09-22 17:47:16 +00:00
// Package s3 provides an interface to Amazon S3 oject storage
2013-06-27 19:13:07 +00:00
package s3
2013-01-08 18:53:35 +00:00
2013-01-23 22:43:20 +00:00
// FIXME need to prevent anything but ListDir working for s3://
2014-12-23 12:09:02 +00:00
/ *
Progress of port to aws - sdk
* Don ' t really need o . meta at all ?
What happens if you CTRL - C a multipart upload
* get an incomplete upload
* disappears when you delete the bucket
* /
2013-01-08 18:53:35 +00:00
import (
"fmt"
"io"
2016-02-01 13:11:27 +00:00
"net/http"
2016-01-27 17:39:33 +00:00
"net/url"
2013-01-08 18:53:35 +00:00
"path"
"regexp"
"strings"
2017-06-07 13:16:50 +00:00
"sync"
2013-01-08 18:53:35 +00:00
"time"
2014-12-23 12:09:02 +00:00
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/awserr"
2015-08-28 07:47:41 +00:00
"github.com/aws/aws-sdk-go/aws/corehandlers"
2014-12-23 12:09:02 +00:00
"github.com/aws/aws-sdk-go/aws/credentials"
2016-02-01 13:11:27 +00:00
"github.com/aws/aws-sdk-go/aws/credentials/ec2rolecreds"
"github.com/aws/aws-sdk-go/aws/ec2metadata"
2015-08-28 07:47:41 +00:00
"github.com/aws/aws-sdk-go/aws/request"
2015-10-30 11:50:45 +00:00
"github.com/aws/aws-sdk-go/aws/session"
2014-12-23 12:09:02 +00:00
"github.com/aws/aws-sdk-go/service/s3"
"github.com/aws/aws-sdk-go/service/s3/s3manager"
2014-03-15 16:06:11 +00:00
"github.com/ncw/rclone/fs"
"github.com/ncw/swift"
2016-06-12 14:06:02 +00:00
"github.com/pkg/errors"
2014-03-15 16:06:11 +00:00
)
2013-06-27 19:13:07 +00:00
// Register with Fs
func init ( ) {
2016-02-18 11:35:25 +00:00
fs . Register ( & fs . RegInfo {
2016-02-15 18:11:53 +00:00
Name : "s3" ,
2016-07-11 11:12:28 +00:00
Description : "Amazon S3 (also Dreamhost, Ceph, Minio)" ,
2016-02-15 18:11:53 +00:00
NewFs : NewFs ,
2014-03-15 16:06:11 +00:00
// AWS endpoints: http://docs.amazonwebservices.com/general/latest/gr/rande.html#s3_region
Options : [ ] fs . Option { {
2016-02-01 13:11:27 +00:00
Name : "env_auth" ,
Help : "Get AWS credentials from runtime (environment variables or EC2 meta data if no env vars). Only applies if access_key_id and secret_access_key is blank." ,
Examples : [ ] fs . OptionExample {
{
Value : "false" ,
Help : "Enter AWS credentials in the next step" ,
} , {
Value : "true" ,
Help : "Get AWS credentials from the environment (env vars or IAM)" ,
} ,
} ,
} , {
2014-03-15 16:06:11 +00:00
Name : "access_key_id" ,
2016-02-01 13:11:27 +00:00
Help : "AWS Access Key ID - leave blank for anonymous access or runtime credentials." ,
2014-03-15 16:06:11 +00:00
} , {
Name : "secret_access_key" ,
2016-02-01 13:11:27 +00:00
Help : "AWS Secret Access Key (password) - leave blank for anonymous access or runtime credentials." ,
2014-03-15 16:06:11 +00:00
} , {
2014-12-23 12:09:02 +00:00
Name : "region" ,
Help : "Region to connect to." ,
2014-03-15 16:06:11 +00:00
Examples : [ ] fs . OptionExample { {
2014-12-23 12:09:02 +00:00
Value : "us-east-1" ,
2014-03-15 16:06:11 +00:00
Help : "The default endpoint - a good choice if you are unsure.\nUS Region, Northern Virginia or Pacific Northwest.\nLeave location constraint empty." ,
2017-04-30 16:11:30 +00:00
} , {
Value : "us-east-2" ,
Help : "US East (Ohio) Region\nNeeds location constraint us-east-2." ,
2014-03-15 16:06:11 +00:00
} , {
2014-12-23 12:09:02 +00:00
Value : "us-west-2" ,
2014-03-15 16:06:11 +00:00
Help : "US West (Oregon) Region\nNeeds location constraint us-west-2." ,
} , {
2014-12-23 12:09:02 +00:00
Value : "us-west-1" ,
2014-03-15 16:06:11 +00:00
Help : "US West (Northern California) Region\nNeeds location constraint us-west-1." ,
2017-02-02 18:59:01 +00:00
} , {
Value : "ca-central-1" ,
Help : "Canada (Central) Region\nNeeds location constraint ca-central-1." ,
2014-03-15 16:06:11 +00:00
} , {
2014-12-23 12:09:02 +00:00
Value : "eu-west-1" ,
2017-04-30 16:11:30 +00:00
Help : "EU (Ireland) Region\nNeeds location constraint EU or eu-west-1." ,
} , {
Value : "eu-west-2" ,
Help : "EU (London) Region\nNeeds location constraint eu-west-2." ,
2014-03-15 16:06:11 +00:00
} , {
2014-12-23 12:09:02 +00:00
Value : "eu-central-1" ,
Help : "EU (Frankfurt) Region\nNeeds location constraint eu-central-1." ,
} , {
Value : "ap-southeast-1" ,
2014-03-15 16:06:11 +00:00
Help : "Asia Pacific (Singapore) Region\nNeeds location constraint ap-southeast-1." ,
} , {
2014-12-23 12:09:02 +00:00
Value : "ap-southeast-2" ,
Help : "Asia Pacific (Sydney) Region\nNeeds location constraint ap-southeast-2." ,
2014-03-15 16:06:11 +00:00
} , {
2014-12-23 12:09:02 +00:00
Value : "ap-northeast-1" ,
2014-03-15 16:06:11 +00:00
Help : "Asia Pacific (Tokyo) Region\nNeeds location constraint ap-northeast-1." ,
2016-07-06 10:14:59 +00:00
} , {
Value : "ap-northeast-2" ,
Help : "Asia Pacific (Seoul)\nNeeds location constraint ap-northeast-2." ,
} , {
Value : "ap-south-1" ,
Help : "Asia Pacific (Mumbai)\nNeeds location constraint ap-south-1." ,
2014-03-15 16:06:11 +00:00
} , {
2014-12-23 12:09:02 +00:00
Value : "sa-east-1" ,
2014-03-15 16:06:11 +00:00
Help : "South America (Sao Paulo) Region\nNeeds location constraint sa-east-1." ,
2014-12-23 12:09:02 +00:00
} , {
Value : "other-v2-signature" ,
2016-02-15 18:11:53 +00:00
Help : "If using an S3 clone that only understands v2 signatures\neg Ceph/Dreamhost\nset this and make sure you set the endpoint." ,
2014-12-23 12:09:02 +00:00
} , {
Value : "other-v4-signature" ,
2016-02-15 18:11:53 +00:00
Help : "If using an S3 clone that understands v4 signatures set this\nand make sure you set the endpoint." ,
2014-03-15 16:06:11 +00:00
} } ,
2014-12-23 12:09:02 +00:00
} , {
Name : "endpoint" ,
Help : "Endpoint for S3 API.\nLeave blank if using AWS to use the default endpoint for the region.\nSpecify if using an S3 clone such as Ceph." ,
2014-03-15 16:06:11 +00:00
} , {
Name : "location_constraint" ,
2014-12-23 12:09:02 +00:00
Help : "Location constraint - must be set to match the Region. Used when creating buckets only." ,
2014-03-15 16:06:11 +00:00
Examples : [ ] fs . OptionExample { {
Value : "" ,
Help : "Empty for US Region, Northern Virginia or Pacific Northwest." ,
2017-04-30 16:11:30 +00:00
} , {
Value : "us-east-2" ,
Help : "US East (Ohio) Region." ,
2014-03-15 16:06:11 +00:00
} , {
Value : "us-west-2" ,
Help : "US West (Oregon) Region." ,
} , {
Value : "us-west-1" ,
Help : "US West (Northern California) Region." ,
2017-02-02 18:59:01 +00:00
} , {
Value : "ca-central-1" ,
Help : "Canada (Central) Region." ,
2014-03-15 16:06:11 +00:00
} , {
Value : "eu-west-1" ,
Help : "EU (Ireland) Region." ,
2017-04-30 16:11:30 +00:00
} , {
Value : "eu-west-2" ,
Help : "EU (London) Region." ,
2014-03-15 16:06:11 +00:00
} , {
Value : "EU" ,
Help : "EU Region." ,
} , {
Value : "ap-southeast-1" ,
Help : "Asia Pacific (Singapore) Region." ,
} , {
Value : "ap-southeast-2" ,
Help : "Asia Pacific (Sydney) Region." ,
} , {
Value : "ap-northeast-1" ,
Help : "Asia Pacific (Tokyo) Region." ,
2016-07-06 10:14:59 +00:00
} , {
Value : "ap-northeast-2" ,
Help : "Asia Pacific (Seoul)" ,
} , {
Value : "ap-south-1" ,
Help : "Asia Pacific (Mumbai)" ,
2014-03-15 16:06:11 +00:00
} , {
Value : "sa-east-1" ,
Help : "South America (Sao Paulo) Region." ,
} } ,
2016-08-22 12:59:03 +00:00
} , {
Name : "acl" ,
2017-03-29 12:38:34 +00:00
Help : "Canned ACL used when creating buckets and/or storing objects in S3.\nFor more info visit https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl" ,
2016-08-22 12:59:03 +00:00
Examples : [ ] fs . OptionExample { {
Value : "private" ,
Help : "Owner gets FULL_CONTROL. No one else has access rights (default)." ,
} , {
Value : "public-read" ,
Help : "Owner gets FULL_CONTROL. The AllUsers group gets READ access." ,
} , {
Value : "public-read-write" ,
Help : "Owner gets FULL_CONTROL. The AllUsers group gets READ and WRITE access.\nGranting this on a bucket is generally not recommended." ,
} , {
Value : "authenticated-read" ,
Help : "Owner gets FULL_CONTROL. The AuthenticatedUsers group gets READ access." ,
} , {
Value : "bucket-owner-read" ,
Help : "Object owner gets FULL_CONTROL. Bucket owner gets READ access.\nIf you specify this canned ACL when creating a bucket, Amazon S3 ignores it." ,
} , {
Value : "bucket-owner-full-control" ,
Help : "Both the object owner and the bucket owner get FULL_CONTROL over the object.\nIf you specify this canned ACL when creating a bucket, Amazon S3 ignores it." ,
} } ,
2016-06-14 20:22:54 +00:00
} , {
Name : "server_side_encryption" ,
Help : "The server-side encryption algorithm used when storing this object in S3." ,
Examples : [ ] fs . OptionExample { {
Value : "" ,
Help : "None" ,
} , {
Value : "AES256" ,
Help : "AES256" ,
} } ,
2016-09-01 21:27:50 +00:00
} , {
Name : "storage_class" ,
Help : "The storage class to use when storing objects in S3." ,
Examples : [ ] fs . OptionExample { {
Value : "" ,
Help : "Default" ,
} , {
Value : "STANDARD" ,
Help : "Standard storage class" ,
} , {
Value : "REDUCED_REDUNDANCY" ,
Help : "Reduced redundancy storage class" ,
} , {
Value : "STANDARD_IA" ,
Help : "Standard Infrequent Access storage class" ,
} } ,
2014-03-15 16:06:11 +00:00
} } ,
} )
2013-06-27 19:13:07 +00:00
}
2013-01-08 18:53:35 +00:00
// Constants
const (
2016-06-19 16:26:44 +00:00
metaMtime = "Mtime" // the meta key to store mtime in - eg X-Amz-Meta-Mtime
2017-09-06 10:13:28 +00:00
listChunkSize = 1000 // number of items to read at once
2016-06-19 16:26:44 +00:00
maxRetries = 10 // number of retries to make of operations
maxSizeForCopy = 5 * 1024 * 1024 * 1024 // The maximum size of object we can COPY
2013-01-08 18:53:35 +00:00
)
2016-09-01 21:27:50 +00:00
// Globals
var (
// Flags
2016-12-20 15:50:46 +00:00
s3ACL = fs . StringP ( "s3-acl" , "" , "" , "Canned ACL used when creating buckets and/or storing objects in S3" )
s3StorageClass = fs . StringP ( "s3-storage-class" , "" , "" , "Storage class to use when uploading S3 objects (STANDARD|REDUCED_REDUNDANCY|STANDARD_IA)" )
2016-09-01 21:27:50 +00:00
)
2015-11-07 11:14:46 +00:00
// Fs represents a remote s3 server
type Fs struct {
2015-10-30 11:50:45 +00:00
name string // the name of the remote
2017-01-13 17:21:47 +00:00
root string // root of the bucket - ignore all objects above this
features * fs . Features // optional features
2015-10-30 11:50:45 +00:00
c * s3 . S3 // the connection to the s3 server
ses * session . Session // the s3 session
bucket string // the bucket we are working on
2017-06-07 13:16:50 +00:00
bucketOKMu sync . Mutex // mutex to protect bucket OK
bucketOK bool // true if we have created the bucket
2017-06-29 11:26:14 +00:00
bucketDeleted bool // true if we have deleted the bucket
2016-08-22 12:59:03 +00:00
acl string // ACL for new buckets / objects
2015-10-30 11:50:45 +00:00
locationConstraint string // location constraint of new buckets
2016-06-14 20:22:54 +00:00
sse string // the type of server-side encryption
2016-09-01 21:27:50 +00:00
storageClass string // storage class
2013-01-08 18:53:35 +00:00
}
2015-11-07 11:14:46 +00:00
// Object describes a s3 object
type Object struct {
2013-01-08 18:53:35 +00:00
// Will definitely have everything but meta which may be nil
//
2016-09-21 21:13:24 +00:00
// List will read everything but meta & mimeType - to fill
// that in you need to call readMetaData
2015-11-07 11:14:46 +00:00
fs * Fs // what this object is part of
2014-12-23 12:09:02 +00:00
remote string // The remote path
etag string // md5sum of the object
bytes int64 // size of the object
lastModified time . Time // Last modified
meta map [ string ] * string // The object metadata if known - may be nil
2016-09-21 21:13:24 +00:00
mimeType string // MimeType of object - may be ""
2013-01-08 18:53:35 +00:00
}
// ------------------------------------------------------------
2015-09-22 17:47:16 +00:00
// Name of the remote (as passed into NewFs)
2015-11-07 11:14:46 +00:00
func ( f * Fs ) Name ( ) string {
2015-08-22 15:53:11 +00:00
return f . name
}
2015-09-22 17:47:16 +00:00
// Root of the remote (as passed into NewFs)
2015-11-07 11:14:46 +00:00
func ( f * Fs ) Root ( ) string {
2015-09-01 19:45:27 +00:00
if f . root == "" {
return f . bucket
}
return f . bucket + "/" + f . root
}
2015-11-07 11:14:46 +00:00
// String converts this Fs to a string
func ( f * Fs ) String ( ) string {
2014-05-05 17:25:32 +00:00
if f . root == "" {
return fmt . Sprintf ( "S3 bucket %s" , f . bucket )
}
return fmt . Sprintf ( "S3 bucket %s path %s" , f . bucket , f . root )
2013-01-08 18:53:35 +00:00
}
2017-01-13 17:21:47 +00:00
// Features returns the optional features of this Fs
func ( f * Fs ) Features ( ) * fs . Features {
return f . features
}
2014-03-15 16:06:11 +00:00
// Pattern to match a s3 path
var matcher = regexp . MustCompile ( ` ^([^/]*)(.*)$ ` )
2013-01-08 18:53:35 +00:00
// parseParse parses a s3 'url'
func s3ParsePath ( path string ) ( bucket , directory string , err error ) {
2014-03-15 16:06:11 +00:00
parts := matcher . FindStringSubmatch ( path )
if parts == nil {
2016-06-12 14:06:02 +00:00
err = errors . Errorf ( "couldn't parse bucket out of s3 path %q" , path )
2013-01-08 18:53:35 +00:00
} else {
2014-03-15 16:06:11 +00:00
bucket , directory = parts [ 1 ] , parts [ 2 ]
2013-01-08 18:53:35 +00:00
directory = strings . Trim ( directory , "/" )
}
return
}
// s3Connection makes a connection to s3
2015-10-30 11:50:45 +00:00
func s3Connection ( name string ) ( * s3 . S3 , * session . Session , error ) {
2013-01-08 18:53:35 +00:00
// Make the auth
2016-02-01 13:11:27 +00:00
v := credentials . Value {
2016-12-20 18:03:09 +00:00
AccessKeyID : fs . ConfigFileGet ( name , "access_key_id" ) ,
SecretAccessKey : fs . ConfigFileGet ( name , "secret_access_key" ) ,
2016-02-01 13:11:27 +00:00
}
// first provider to supply a credential set "wins"
providers := [ ] credentials . Provider {
// use static credentials if they're present (checked by provider)
& credentials . StaticProvider { Value : v } ,
// * Access Key ID: AWS_ACCESS_KEY_ID or AWS_ACCESS_KEY
// * Secret Access Key: AWS_SECRET_ACCESS_KEY or AWS_SECRET_KEY
& credentials . EnvProvider { } ,
// Pick up IAM role in case we're on EC2
& ec2rolecreds . EC2RoleProvider {
Client : ec2metadata . New ( session . New ( ) , & aws . Config {
HTTPClient : & http . Client { Timeout : 1 * time . Second } , // low timeout to ec2 metadata service
} ) ,
ExpiryWindow : 3 ,
} ,
}
cred := credentials . NewChainCredentials ( providers )
2015-09-29 08:58:03 +00:00
switch {
2016-12-20 18:03:09 +00:00
case fs . ConfigFileGetBool ( name , "env_auth" , false ) :
2016-02-09 17:19:33 +00:00
// No need for empty checks if "env_auth" is true
case v . AccessKeyID == "" && v . SecretAccessKey == "" :
2016-02-01 13:11:27 +00:00
// if no access key/secret and iam is explicitly disabled then fall back to anon interaction
cred = credentials . AnonymousCredentials
case v . AccessKeyID == "" :
2015-10-30 11:50:45 +00:00
return nil , nil , errors . New ( "access_key_id not found" )
2016-02-01 13:11:27 +00:00
case v . SecretAccessKey == "" :
2015-10-30 11:50:45 +00:00
return nil , nil , errors . New ( "secret_access_key not found" )
2013-01-08 18:53:35 +00:00
}
2016-12-20 18:03:09 +00:00
endpoint := fs . ConfigFileGet ( name , "endpoint" )
region := fs . ConfigFileGet ( name , "region" )
2014-12-23 12:09:02 +00:00
if region == "" && endpoint == "" {
endpoint = "https://s3.amazonaws.com/"
2013-01-08 18:53:35 +00:00
}
2014-12-23 12:09:02 +00:00
if region == "" {
region = "us-east-1"
2013-01-08 18:53:35 +00:00
}
2014-12-23 12:09:02 +00:00
awsConfig := aws . NewConfig ( ) .
WithRegion ( region ) .
WithMaxRetries ( maxRetries ) .
2016-02-01 13:11:27 +00:00
WithCredentials ( cred ) .
2014-12-23 12:09:02 +00:00
WithEndpoint ( endpoint ) .
2015-08-10 10:02:34 +00:00
WithHTTPClient ( fs . Config . Client ( ) ) .
WithS3ForcePathStyle ( true )
// awsConfig.WithLogLevel(aws.LogDebugWithSigning)
2015-10-30 11:50:45 +00:00
ses := session . New ( )
c := s3 . New ( ses , awsConfig )
2014-12-23 12:09:02 +00:00
if region == "other-v2-signature" {
2017-02-09 11:01:20 +00:00
fs . Debugf ( name , "Using v2 auth" )
2015-08-28 07:47:41 +00:00
signer := func ( req * request . Request ) {
2015-08-10 10:02:34 +00:00
// Ignore AnonymousCredentials object
2015-10-30 11:50:45 +00:00
if req . Config . Credentials == credentials . AnonymousCredentials {
2015-08-10 10:02:34 +00:00
return
}
2016-02-01 13:11:27 +00:00
sign ( v . AccessKeyID , v . SecretAccessKey , req . HTTPRequest )
2015-08-10 10:02:34 +00:00
}
c . Handlers . Sign . Clear ( )
2015-08-28 07:47:41 +00:00
c . Handlers . Sign . PushBackNamed ( corehandlers . BuildContentLengthHandler )
2015-08-10 10:02:34 +00:00
c . Handlers . Sign . PushBack ( signer )
2013-01-08 18:53:35 +00:00
}
2015-10-30 11:50:45 +00:00
return c , ses , nil
2013-01-08 18:53:35 +00:00
}
2016-09-01 21:27:50 +00:00
// NewFs constructs an Fs from the path, bucket:path
2014-05-05 18:52:52 +00:00
func NewFs ( name , root string ) ( fs . Fs , error ) {
bucket , directory , err := s3ParsePath ( root )
2013-01-08 18:53:35 +00:00
if err != nil {
return nil , err
}
2015-10-30 11:50:45 +00:00
c , ses , err := s3Connection ( name )
2013-01-08 18:53:35 +00:00
if err != nil {
return nil , err
}
2015-11-07 11:14:46 +00:00
f := & Fs {
2016-08-22 12:59:03 +00:00
name : name ,
c : c ,
bucket : bucket ,
ses : ses ,
2016-12-20 18:03:09 +00:00
acl : fs . ConfigFileGet ( name , "acl" ) ,
2014-12-23 12:09:02 +00:00
root : directory ,
2016-12-20 18:03:09 +00:00
locationConstraint : fs . ConfigFileGet ( name , "location_constraint" ) ,
sse : fs . ConfigFileGet ( name , "server_side_encryption" ) ,
storageClass : fs . ConfigFileGet ( name , "storage_class" ) ,
2016-09-01 21:27:50 +00:00
}
2017-08-09 14:27:43 +00:00
f . features = ( & fs . Features {
ReadMimeType : true ,
WriteMimeType : true ,
BucketBased : true ,
} ) . Fill ( f )
2016-11-03 21:05:30 +00:00
if * s3ACL != "" {
f . acl = * s3ACL
}
2016-09-01 21:27:50 +00:00
if * s3StorageClass != "" {
f . storageClass = * s3StorageClass
2013-01-08 18:53:35 +00:00
}
2014-05-05 18:52:52 +00:00
if f . root != "" {
f . root += "/"
// Check to see if the object exists
2014-12-23 12:09:02 +00:00
req := s3 . HeadObjectInput {
Bucket : & f . bucket ,
Key : & directory ,
}
_ , err = f . c . HeadObject ( & req )
2014-05-05 18:52:52 +00:00
if err == nil {
f . root = path . Dir ( directory )
if f . root == "." {
f . root = ""
} else {
f . root += "/"
}
2016-06-21 17:01:53 +00:00
// return an error with an fs which points to the parent
return f , fs . ErrorIsFile
2014-05-05 18:52:52 +00:00
}
}
2014-12-23 12:09:02 +00:00
// f.listMultipartUploads()
2013-01-08 18:53:35 +00:00
return f , nil
}
2016-06-25 20:58:34 +00:00
// Return an Object from a path
2013-01-08 18:53:35 +00:00
//
2016-06-25 20:23:20 +00:00
//If it can't be found it returns the error ErrorObjectNotFound.
func ( f * Fs ) newObjectWithInfo ( remote string , info * s3 . Object ) ( fs . Object , error ) {
2015-11-07 11:14:46 +00:00
o := & Object {
fs : f ,
2013-01-08 18:53:35 +00:00
remote : remote ,
}
if info != nil {
// Set info but not meta
2014-12-23 12:09:02 +00:00
if info . LastModified == nil {
2017-02-09 11:01:20 +00:00
fs . Logf ( o , "Failed to read last modified" )
2013-06-27 19:13:07 +00:00
o . lastModified = time . Now ( )
2014-12-23 12:09:02 +00:00
} else {
o . lastModified = * info . LastModified
2013-01-08 18:53:35 +00:00
}
2014-12-23 12:09:02 +00:00
o . etag = aws . StringValue ( info . ETag )
o . bytes = aws . Int64Value ( info . Size )
2013-01-08 18:53:35 +00:00
} else {
2013-06-27 19:13:07 +00:00
err := o . readMetaData ( ) // reads info and meta, returning an error
2013-01-08 18:53:35 +00:00
if err != nil {
2016-06-25 20:23:20 +00:00
return nil , err
2013-01-08 18:53:35 +00:00
}
}
2016-06-25 20:23:20 +00:00
return o , nil
2013-01-08 18:53:35 +00:00
}
2016-06-25 20:23:20 +00:00
// NewObject finds the Object at remote. If it can't be found
// it returns the error fs.ErrorObjectNotFound.
func ( f * Fs ) NewObject ( remote string ) ( fs . Object , error ) {
2016-06-25 20:58:34 +00:00
return f . newObjectWithInfo ( remote , nil )
2013-01-08 18:53:35 +00:00
}
2016-04-21 19:06:21 +00:00
// listFn is called from list to handle an object.
type listFn func ( remote string , object * s3 . Object , isDirectory bool ) error
2014-05-05 17:25:32 +00:00
// list the objects into the function supplied
//
2016-04-23 20:46:52 +00:00
// dir is the starting directory, "" for root
//
2017-06-11 21:43:31 +00:00
// Set recurse to read sub directories
func ( f * Fs ) list ( dir string , recurse bool , fn listFn ) error {
2016-04-23 20:46:52 +00:00
root := f . root
if dir != "" {
root += dir + "/"
}
2014-12-23 12:09:02 +00:00
maxKeys := int64 ( listChunkSize )
2014-05-05 17:25:32 +00:00
delimiter := ""
2017-06-11 21:43:31 +00:00
if ! recurse {
2014-05-05 17:25:32 +00:00
delimiter = "/"
}
2014-12-23 12:09:02 +00:00
var marker * string
2015-02-10 17:58:29 +00:00
for {
2014-12-23 12:09:02 +00:00
// FIXME need to implement ALL loop
req := s3 . ListObjectsInput {
Bucket : & f . bucket ,
Delimiter : & delimiter ,
2016-04-23 20:46:52 +00:00
Prefix : & root ,
2014-12-23 12:09:02 +00:00
MaxKeys : & maxKeys ,
Marker : marker ,
}
resp , err := f . c . ListObjects ( & req )
2015-02-10 17:58:29 +00:00
if err != nil {
2017-06-11 21:43:31 +00:00
if awsErr , ok := err . ( awserr . RequestFailure ) ; ok {
if awsErr . StatusCode ( ) == http . StatusNotFound {
err = fs . ErrorDirNotFound
}
}
2016-04-21 19:06:21 +00:00
return err
}
rootLength := len ( f . root )
2017-06-11 21:43:31 +00:00
if ! recurse {
2016-04-21 19:06:21 +00:00
for _ , commonPrefix := range resp . CommonPrefixes {
if commonPrefix . Prefix == nil {
2017-02-09 11:01:20 +00:00
fs . Logf ( f , "Nil common prefix received" )
2016-04-21 19:06:21 +00:00
continue
2015-02-10 17:58:29 +00:00
}
2016-04-21 19:06:21 +00:00
remote := * commonPrefix . Prefix
if ! strings . HasPrefix ( remote , f . root ) {
2017-02-09 11:01:20 +00:00
fs . Logf ( f , "Odd name received %q" , remote )
2016-04-21 19:06:21 +00:00
continue
}
remote = remote [ rootLength : ]
if strings . HasSuffix ( remote , "/" ) {
remote = remote [ : len ( remote ) - 1 ]
}
err = fn ( remote , & s3 . Object { Key : & remote } , true )
if err != nil {
return err
2013-01-08 18:53:35 +00:00
}
}
2016-04-21 19:06:21 +00:00
}
for _ , object := range resp . Contents {
key := aws . StringValue ( object . Key )
if ! strings . HasPrefix ( key , f . root ) {
2017-02-09 11:01:20 +00:00
fs . Logf ( f , "Odd name received %q" , key )
2016-04-21 19:06:21 +00:00
continue
2014-12-23 12:09:02 +00:00
}
2016-04-21 19:06:21 +00:00
remote := key [ rootLength : ]
err = fn ( remote , object , false )
if err != nil {
return err
2014-12-23 12:09:02 +00:00
}
2015-02-10 17:58:29 +00:00
}
2016-04-21 19:06:21 +00:00
if ! aws . BoolValue ( resp . IsTruncated ) {
break
}
// Use NextMarker if set, otherwise use last Key
if resp . NextMarker == nil || * resp . NextMarker == "" {
marker = resp . Contents [ len ( resp . Contents ) - 1 ] . Key
} else {
marker = resp . NextMarker
}
2014-05-05 17:25:32 +00:00
}
2016-04-21 19:06:21 +00:00
return nil
2014-05-05 17:25:32 +00:00
}
2017-06-30 09:54:14 +00:00
// Convert a list item into a DirEntry
func ( f * Fs ) itemToDirEntry ( remote string , object * s3 . Object , isDirectory bool ) ( fs . DirEntry , error ) {
2017-06-11 21:43:31 +00:00
if isDirectory {
size := int64 ( 0 )
if object . Size != nil {
size = * object . Size
}
2017-06-30 12:37:29 +00:00
d := fs . NewDir ( remote , time . Time { } ) . SetSize ( size )
2017-06-11 21:43:31 +00:00
return d , nil
2016-04-21 19:06:21 +00:00
}
2017-06-11 21:43:31 +00:00
o , err := f . newObjectWithInfo ( remote , object )
if err != nil {
return nil , err
}
return o , nil
}
// listDir lists files and directories to out
func ( f * Fs ) listDir ( dir string ) ( entries fs . DirEntries , err error ) {
2016-04-21 19:06:21 +00:00
// List the objects and directories
2017-06-11 21:43:31 +00:00
err = f . list ( dir , false , func ( remote string , object * s3 . Object , isDirectory bool ) error {
entry , err := f . itemToDirEntry ( remote , object , isDirectory )
if err != nil {
return err
}
if entry != nil {
entries = append ( entries , entry )
2016-04-21 19:06:21 +00:00
}
return nil
} )
if err != nil {
2017-06-11 21:43:31 +00:00
return nil , err
2016-04-21 19:06:21 +00:00
}
2017-06-11 21:43:31 +00:00
return entries , nil
2016-04-21 19:06:21 +00:00
}
// listBuckets lists the buckets to out
2017-06-11 21:43:31 +00:00
func ( f * Fs ) listBuckets ( dir string ) ( entries fs . DirEntries , err error ) {
2016-04-23 20:46:52 +00:00
if dir != "" {
2017-06-11 21:43:31 +00:00
return nil , fs . ErrorListBucketRequired
2016-04-23 20:46:52 +00:00
}
2016-04-21 19:06:21 +00:00
req := s3 . ListBucketsInput { }
resp , err := f . c . ListBuckets ( & req )
if err != nil {
2017-06-11 21:43:31 +00:00
return nil , err
2016-04-21 19:06:21 +00:00
}
for _ , bucket := range resp . Buckets {
2017-06-30 12:37:29 +00:00
d := fs . NewDir ( aws . StringValue ( bucket . Name ) , aws . TimeValue ( bucket . CreationDate ) )
2017-06-11 21:43:31 +00:00
entries = append ( entries , d )
2014-05-05 17:25:32 +00:00
}
2017-06-11 21:43:31 +00:00
return entries , nil
2013-01-08 18:53:35 +00:00
}
2017-06-11 21:43:31 +00:00
// List the objects and directories in dir into entries. The
// entries can be returned in any order but should be for a
// complete directory.
//
// dir should be "" to list the root, and should not have
// trailing slashes.
//
// This should return ErrDirNotFound if the directory isn't
// found.
func ( f * Fs ) List ( dir string ) ( entries fs . DirEntries , err error ) {
2014-05-05 17:25:32 +00:00
if f . bucket == "" {
2017-06-11 21:43:31 +00:00
return f . listBuckets ( dir )
2014-05-05 17:25:32 +00:00
}
2017-06-11 21:43:31 +00:00
return f . listDir ( dir )
2013-01-23 22:43:20 +00:00
}
2017-06-05 15:14:24 +00:00
// ListR lists the objects and directories of the Fs starting
// from dir recursively into out.
2017-06-11 21:43:31 +00:00
//
// dir should be "" to start from the root, and should not
// have trailing slashes.
//
// This should return ErrDirNotFound if the directory isn't
// found.
//
// It should call callback for each tranche of entries read.
// These need not be returned in any particular order. If
// callback returns an error then the listing will stop
// immediately.
//
// Don't implement this unless you have a more efficient way
// of listing recursively that doing a directory traversal.
func ( f * Fs ) ListR ( dir string , callback fs . ListRCallback ) ( err error ) {
if f . bucket == "" {
return fs . ErrorListBucketRequired
}
list := fs . NewListRHelper ( callback )
err = f . list ( dir , true , func ( remote string , object * s3 . Object , isDirectory bool ) error {
entry , err := f . itemToDirEntry ( remote , object , isDirectory )
if err != nil {
return err
}
return list . Add ( entry )
} )
if err != nil {
return err
}
return list . Flush ( )
2017-06-05 15:14:24 +00:00
}
2016-06-25 20:58:34 +00:00
// Put the Object into the bucket
2017-05-28 11:44:22 +00:00
func ( f * Fs ) Put ( in io . Reader , src fs . ObjectInfo , options ... fs . OpenOption ) ( fs . Object , error ) {
2015-11-07 11:14:46 +00:00
// Temporary Object under construction
fs := & Object {
fs : f ,
2016-02-18 11:35:25 +00:00
remote : src . Remote ( ) ,
2015-11-07 11:14:46 +00:00
}
2017-05-28 11:44:22 +00:00
return fs , fs . Update ( in , src , options ... )
2013-01-08 18:53:35 +00:00
}
2016-02-24 01:58:55 +00:00
// Check if the bucket exists
2017-06-29 11:26:14 +00:00
//
// NB this can return incorrect results if called immediately after bucket deletion
2016-02-24 01:58:55 +00:00
func ( f * Fs ) dirExists ( ) ( bool , error ) {
req := s3 . HeadBucketInput {
Bucket : & f . bucket ,
}
_ , err := f . c . HeadBucket ( & req )
if err == nil {
return true , nil
}
if err , ok := err . ( awserr . RequestFailure ) ; ok {
if err . StatusCode ( ) == http . StatusNotFound {
return false , nil
}
}
return false , err
}
2013-01-08 18:53:35 +00:00
// Mkdir creates the bucket if it doesn't exist
2016-11-25 21:52:43 +00:00
func ( f * Fs ) Mkdir ( dir string ) error {
2017-06-07 13:16:50 +00:00
f . bucketOKMu . Lock ( )
defer f . bucketOKMu . Unlock ( )
if f . bucketOK {
2016-11-25 21:52:43 +00:00
return nil
}
2017-06-29 11:26:14 +00:00
if ! f . bucketDeleted {
exists , err := f . dirExists ( )
if err == nil {
f . bucketOK = exists
}
if err != nil || exists {
return err
}
2016-02-24 01:58:55 +00:00
}
2014-12-23 12:09:02 +00:00
req := s3 . CreateBucketInput {
Bucket : & f . bucket ,
2016-08-22 12:59:03 +00:00
ACL : & f . acl ,
2014-12-23 12:09:02 +00:00
}
if f . locationConstraint != "" {
req . CreateBucketConfiguration = & s3 . CreateBucketConfiguration {
LocationConstraint : & f . locationConstraint ,
}
}
2017-06-29 11:26:14 +00:00
_ , err := f . c . CreateBucket ( & req )
2014-12-23 12:09:02 +00:00
if err , ok := err . ( awserr . Error ) ; ok {
if err . Code ( ) == "BucketAlreadyOwnedByYou" {
2017-06-07 13:16:50 +00:00
err = nil
2013-01-08 22:31:16 +00:00
}
}
2017-06-07 13:16:50 +00:00
if err == nil {
f . bucketOK = true
2017-06-29 11:26:14 +00:00
f . bucketDeleted = false
2017-06-07 13:16:50 +00:00
}
2013-01-08 22:31:16 +00:00
return err
2013-01-08 18:53:35 +00:00
}
2015-11-07 15:31:04 +00:00
// Rmdir deletes the bucket if the fs is at the root
2013-01-08 18:53:35 +00:00
//
// Returns an error if it isn't empty
2016-11-25 21:52:43 +00:00
func ( f * Fs ) Rmdir ( dir string ) error {
2017-06-07 13:16:50 +00:00
f . bucketOKMu . Lock ( )
defer f . bucketOKMu . Unlock ( )
2016-11-25 21:52:43 +00:00
if f . root != "" || dir != "" {
2015-11-07 15:31:04 +00:00
return nil
}
2014-12-23 12:09:02 +00:00
req := s3 . DeleteBucketInput {
Bucket : & f . bucket ,
}
_ , err := f . c . DeleteBucket ( & req )
2017-06-07 13:16:50 +00:00
if err == nil {
f . bucketOK = false
2017-06-29 11:26:14 +00:00
f . bucketDeleted = true
2017-06-07 13:16:50 +00:00
}
2014-12-23 12:09:02 +00:00
return err
2013-01-08 18:53:35 +00:00
}
2015-09-22 17:47:16 +00:00
// Precision of the remote
2015-11-07 11:14:46 +00:00
func ( f * Fs ) Precision ( ) time . Duration {
2013-01-18 23:21:02 +00:00
return time . Nanosecond
}
2015-02-14 18:48:08 +00:00
// Copy src to this remote using server side copy operations.
//
// This is stored with the remote path given
//
// It returns the destination Object and a possible error
//
// Will only be called if src.Fs().Name() == f.Name()
//
// If it isn't possible then return fs.ErrorCantCopy
2015-11-07 11:14:46 +00:00
func ( f * Fs ) Copy ( src fs . Object , remote string ) ( fs . Object , error ) {
2017-06-28 20:14:53 +00:00
err := f . Mkdir ( "" )
if err != nil {
return nil , err
}
2015-11-07 11:14:46 +00:00
srcObj , ok := src . ( * Object )
2015-02-14 18:48:08 +00:00
if ! ok {
2017-02-09 11:01:20 +00:00
fs . Debugf ( src , "Can't copy - not same remote type" )
2015-02-14 18:48:08 +00:00
return nil , fs . ErrorCantCopy
}
2015-11-07 11:14:46 +00:00
srcFs := srcObj . fs
2015-02-14 18:48:08 +00:00
key := f . root + remote
2016-01-27 17:39:33 +00:00
source := url . QueryEscape ( srcFs . bucket + "/" + srcFs . root + srcObj . remote )
2015-02-14 18:48:08 +00:00
req := s3 . CopyObjectInput {
Bucket : & f . bucket ,
Key : & key ,
CopySource : & source ,
MetadataDirective : aws . String ( s3 . MetadataDirectiveCopy ) ,
}
2017-06-28 20:14:53 +00:00
_ , err = f . c . CopyObject ( & req )
2015-02-14 18:48:08 +00:00
if err != nil {
return nil , err
}
2016-06-25 20:23:20 +00:00
return f . NewObject ( remote )
2015-02-14 18:48:08 +00:00
}
2016-01-11 12:39:33 +00:00
// Hashes returns the supported hash sets.
func ( f * Fs ) Hashes ( ) fs . HashSet {
return fs . HashSet ( fs . HashMD5 )
}
2013-01-08 18:53:35 +00:00
// ------------------------------------------------------------
2015-09-22 17:47:16 +00:00
// Fs returns the parent Fs
2016-02-18 11:35:25 +00:00
func ( o * Object ) Fs ( ) fs . Info {
2015-11-07 11:14:46 +00:00
return o . fs
2014-03-28 17:56:04 +00:00
}
// Return a string version
2015-11-07 11:14:46 +00:00
func ( o * Object ) String ( ) string {
2014-03-28 17:56:04 +00:00
if o == nil {
return "<nil>"
}
return o . remote
}
2015-09-22 17:47:16 +00:00
// Remote returns the remote path
2015-11-07 11:14:46 +00:00
func ( o * Object ) Remote ( ) string {
2013-06-27 19:13:07 +00:00
return o . remote
2013-01-08 18:53:35 +00:00
}
2015-05-09 09:37:43 +00:00
var matchMd5 = regexp . MustCompile ( ` ^[0-9a-f] { 32}$ ` )
2016-01-11 12:39:33 +00:00
// Hash returns the Md5sum of an object returning a lowercase hex string
func ( o * Object ) Hash ( t fs . HashType ) ( string , error ) {
if t != fs . HashMD5 {
return "" , fs . ErrHashUnsupported
}
2015-05-09 09:37:43 +00:00
etag := strings . Trim ( strings . ToLower ( o . etag ) , ` " ` )
// Check the etag is a valid md5sum
if ! matchMd5 . MatchString ( etag ) {
2017-02-09 11:01:20 +00:00
// fs.Debugf(o, "Invalid md5sum (probably multipart uploaded) - ignoring: %q", etag)
2015-05-09 09:37:43 +00:00
return "" , nil
}
return etag , nil
2013-01-08 18:53:35 +00:00
}
// Size returns the size of an object in bytes
2015-11-07 11:14:46 +00:00
func ( o * Object ) Size ( ) int64 {
2013-06-27 19:13:07 +00:00
return o . bytes
2013-01-08 18:53:35 +00:00
}
// readMetaData gets the metadata if it hasn't already been fetched
//
// it also sets the info
2015-11-07 11:14:46 +00:00
func ( o * Object ) readMetaData ( ) ( err error ) {
2013-06-27 19:13:07 +00:00
if o . meta != nil {
2013-01-08 18:53:35 +00:00
return nil
}
2015-11-07 11:14:46 +00:00
key := o . fs . root + o . remote
2014-12-23 12:09:02 +00:00
req := s3 . HeadObjectInput {
2015-11-07 11:14:46 +00:00
Bucket : & o . fs . bucket ,
2014-12-23 12:09:02 +00:00
Key : & key ,
2014-07-28 21:32:15 +00:00
}
2015-11-07 11:14:46 +00:00
resp , err := o . fs . c . HeadObject ( & req )
2013-01-08 18:53:35 +00:00
if err != nil {
2016-06-25 20:23:20 +00:00
if awsErr , ok := err . ( awserr . RequestFailure ) ; ok {
if awsErr . StatusCode ( ) == http . StatusNotFound {
return fs . ErrorObjectNotFound
}
}
2013-01-08 18:53:35 +00:00
return err
}
2014-05-16 15:27:53 +00:00
var size int64
// Ignore missing Content-Length assuming it is 0
// Some versions of ceph do this due their apache proxies
2014-12-23 12:09:02 +00:00
if resp . ContentLength != nil {
size = * resp . ContentLength
2013-01-08 18:53:35 +00:00
}
2014-12-23 12:09:02 +00:00
o . etag = aws . StringValue ( resp . ETag )
2013-06-27 19:13:07 +00:00
o . bytes = size
2014-12-23 12:09:02 +00:00
o . meta = resp . Metadata
if resp . LastModified == nil {
2017-02-09 11:01:20 +00:00
fs . Logf ( o , "Failed to read last modified from HEAD: %v" , err )
2013-06-27 19:13:07 +00:00
o . lastModified = time . Now ( )
2014-12-23 12:09:02 +00:00
} else {
o . lastModified = * resp . LastModified
2013-01-08 18:53:35 +00:00
}
2016-09-21 21:13:24 +00:00
o . mimeType = aws . StringValue ( resp . ContentType )
2013-01-08 18:53:35 +00:00
return nil
}
// ModTime returns the modification time of the object
//
// It attempts to read the objects mtime and if that isn't present the
// LastModified returned in the http headers
2015-11-07 11:14:46 +00:00
func ( o * Object ) ModTime ( ) time . Time {
2013-06-27 19:13:07 +00:00
err := o . readMetaData ( )
2013-01-08 18:53:35 +00:00
if err != nil {
2017-02-09 11:01:20 +00:00
fs . Logf ( o , "Failed to read metadata: %v" , err )
2013-01-08 18:53:35 +00:00
return time . Now ( )
}
// read mtime out of metadata if available
2013-06-27 19:13:07 +00:00
d , ok := o . meta [ metaMtime ]
2014-12-23 12:09:02 +00:00
if ! ok || d == nil {
2017-02-09 11:01:20 +00:00
// fs.Debugf(o, "No metadata")
2013-06-27 19:13:07 +00:00
return o . lastModified
2013-01-08 18:53:35 +00:00
}
2014-12-23 12:09:02 +00:00
modTime , err := swift . FloatStringToTime ( * d )
2013-01-08 18:53:35 +00:00
if err != nil {
2017-02-09 11:01:20 +00:00
fs . Logf ( o , "Failed to read mtime from object: %v" , err )
2013-06-27 19:13:07 +00:00
return o . lastModified
2013-01-08 18:53:35 +00:00
}
return modTime
}
2015-09-22 17:47:16 +00:00
// SetModTime sets the modification time of the local fs object
2016-03-22 15:07:10 +00:00
func ( o * Object ) SetModTime ( modTime time . Time ) error {
2013-06-27 19:13:07 +00:00
err := o . readMetaData ( )
2013-01-08 22:31:16 +00:00
if err != nil {
2016-03-22 15:07:10 +00:00
return err
2013-01-08 22:31:16 +00:00
}
2014-12-23 12:09:02 +00:00
o . meta [ metaMtime ] = aws . String ( swift . TimeToFloatString ( modTime ) )
2016-06-19 16:26:44 +00:00
if o . bytes >= maxSizeForCopy {
2017-02-09 11:01:20 +00:00
fs . Debugf ( o , "SetModTime is unsupported for objects bigger than %v bytes" , fs . SizeSuffix ( maxSizeForCopy ) )
2016-06-19 16:26:44 +00:00
return nil
}
2016-01-02 08:58:48 +00:00
// Guess the content type
2016-09-21 21:13:24 +00:00
mimeType := fs . MimeType ( o )
2016-01-02 08:58:48 +00:00
2014-12-23 12:09:02 +00:00
// Copy the object to itself to update the metadata
2015-11-07 11:14:46 +00:00
key := o . fs . root + o . remote
sourceKey := o . fs . bucket + "/" + key
2014-12-23 12:09:02 +00:00
directive := s3 . MetadataDirectiveReplace // replace metadata with that passed in
req := s3 . CopyObjectInput {
2015-11-07 11:14:46 +00:00
Bucket : & o . fs . bucket ,
2016-08-22 12:59:03 +00:00
ACL : & o . fs . acl ,
2014-12-23 12:09:02 +00:00
Key : & key ,
2016-09-21 21:13:24 +00:00
ContentType : & mimeType ,
2016-01-27 17:39:33 +00:00
CopySource : aws . String ( url . QueryEscape ( sourceKey ) ) ,
2014-12-23 12:09:02 +00:00
Metadata : o . meta ,
MetadataDirective : & directive ,
}
2015-11-07 11:14:46 +00:00
_ , err = o . fs . c . CopyObject ( & req )
2016-03-22 15:07:10 +00:00
return err
2013-01-08 18:53:35 +00:00
}
2015-09-22 17:47:16 +00:00
// Storable raturns a boolean indicating if this object is storable
2015-11-07 11:14:46 +00:00
func ( o * Object ) Storable ( ) bool {
2013-01-08 18:53:35 +00:00
return true
}
// Open an object for read
2016-09-10 10:29:57 +00:00
func ( o * Object ) Open ( options ... fs . OpenOption ) ( in io . ReadCloser , err error ) {
2015-11-07 11:14:46 +00:00
key := o . fs . root + o . remote
2014-12-23 12:09:02 +00:00
req := s3 . GetObjectInput {
2015-11-07 11:14:46 +00:00
Bucket : & o . fs . bucket ,
2014-12-23 12:09:02 +00:00
Key : & key ,
}
2016-09-10 10:29:57 +00:00
for _ , option := range options {
switch option . ( type ) {
case * fs . RangeOption , * fs . SeekOption :
_ , value := option . Header ( )
req . Range = & value
default :
if option . Mandatory ( ) {
2017-02-09 11:01:20 +00:00
fs . Logf ( o , "Unsupported mandatory option: %v" , option )
2016-09-10 10:29:57 +00:00
}
}
}
2015-11-07 11:14:46 +00:00
resp , err := o . fs . c . GetObject ( & req )
2017-09-09 12:02:26 +00:00
if err , ok := err . ( awserr . RequestFailure ) ; ok {
if err . Code ( ) == "InvalidObjectState" {
return nil , errors . Errorf ( "Object in GLACIER, restore first: %v" , key )
}
}
2014-12-23 12:09:02 +00:00
if err != nil {
return nil , err
}
return resp . Body , nil
2013-01-08 18:53:35 +00:00
}
2014-04-18 16:04:21 +00:00
// Update the Object from in with modTime and size
2017-05-28 11:44:22 +00:00
func ( o * Object ) Update ( in io . Reader , src fs . ObjectInfo , options ... fs . OpenOption ) error {
2017-06-07 13:16:50 +00:00
err := o . fs . Mkdir ( "" )
if err != nil {
return err
}
2016-02-18 11:35:25 +00:00
modTime := src . ModTime ( )
2015-11-07 11:14:46 +00:00
uploader := s3manager . NewUploader ( o . fs . ses , func ( u * s3manager . Uploader ) {
2015-10-30 11:50:45 +00:00
u . Concurrency = 2
u . LeavePartsOnError = false
2015-11-07 11:14:46 +00:00
u . S3 = o . fs . c
2016-03-10 16:48:55 +00:00
u . PartSize = s3manager . MinUploadPartSize
size := src . Size ( )
// Adjust PartSize until the number of parts is small enough.
if size / u . PartSize >= s3manager . MaxUploadParts {
// Calculate partition size rounded up to the nearest MB
u . PartSize = ( ( ( size / s3manager . MaxUploadParts ) >> 20 ) + 1 ) << 20
}
2015-10-30 11:50:45 +00:00
} )
2014-04-18 16:04:21 +00:00
2014-12-23 12:09:02 +00:00
// Set the mtime in the meta data
metadata := map [ string ] * string {
metaMtime : aws . String ( swift . TimeToFloatString ( modTime ) ) ,
}
// Guess the content type
2016-09-21 21:13:24 +00:00
mimeType := fs . MimeType ( src )
2014-12-23 12:09:02 +00:00
2015-11-07 11:14:46 +00:00
key := o . fs . root + o . remote
2014-12-23 12:09:02 +00:00
req := s3manager . UploadInput {
2015-11-07 11:14:46 +00:00
Bucket : & o . fs . bucket ,
2016-08-22 12:59:03 +00:00
ACL : & o . fs . acl ,
2014-12-23 12:09:02 +00:00
Key : & key ,
Body : in ,
2016-09-21 21:13:24 +00:00
ContentType : & mimeType ,
2014-12-23 12:09:02 +00:00
Metadata : metadata ,
//ContentLength: &size,
}
2016-06-14 20:22:54 +00:00
if o . fs . sse != "" {
req . ServerSideEncryption = & o . fs . sse
}
2016-09-01 21:27:50 +00:00
if o . fs . storageClass != "" {
req . StorageClass = & o . fs . storageClass
}
2017-06-07 13:16:50 +00:00
_ , err = uploader . Upload ( & req )
2014-07-19 11:37:11 +00:00
if err != nil {
return err
}
2014-12-23 12:09:02 +00:00
2014-07-19 11:37:11 +00:00
// Read the metadata from the newly created object
2014-07-20 10:23:05 +00:00
o . meta = nil // wipe old metadata
2014-07-19 11:37:11 +00:00
err = o . readMetaData ( )
2014-04-18 16:04:21 +00:00
return err
}
2013-01-08 18:53:35 +00:00
// Remove an object
2015-11-07 11:14:46 +00:00
func ( o * Object ) Remove ( ) error {
key := o . fs . root + o . remote
2014-12-23 12:09:02 +00:00
req := s3 . DeleteObjectInput {
2015-11-07 11:14:46 +00:00
Bucket : & o . fs . bucket ,
2014-12-23 12:09:02 +00:00
Key : & key ,
}
2015-11-07 11:14:46 +00:00
_ , err := o . fs . c . DeleteObject ( & req )
2014-12-23 12:09:02 +00:00
return err
2013-01-08 18:53:35 +00:00
}
2016-09-21 21:13:24 +00:00
// MimeType of an Object if known, "" otherwise
func ( o * Object ) MimeType ( ) string {
err := o . readMetaData ( )
if err != nil {
2017-02-09 11:01:20 +00:00
fs . Logf ( o , "Failed to read metadata: %v" , err )
2016-09-21 21:13:24 +00:00
return ""
}
return o . mimeType
}
2013-01-08 18:53:35 +00:00
// Check the interfaces are satisfied
2015-11-07 11:14:46 +00:00
var (
2016-09-21 21:13:24 +00:00
_ fs . Fs = & Fs { }
_ fs . Copier = & Fs { }
2017-06-05 15:14:24 +00:00
_ fs . ListRer = & Fs { }
2016-09-21 21:13:24 +00:00
_ fs . Object = & Object { }
_ fs . MimeTyper = & Object { }
2015-11-07 11:14:46 +00:00
)