KurlesHS/rclone
1
0
Fork 0
forked from TrueCloudLab/rclone
Code Pull requests Activity

s3: allow streaming upload of files with unknown file size (see #1614)

Browse source
This commit is contained in:
Stefan Breunig 2017-09-15 20:20:32 +02:00
parent 9d3f11b493
commit 7f8d306c9c
1 changed files with 22 additions and 9 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

31
s3/s3.go
View file

@ -219,10 +219,11 @@ func init() {
// Constants // Constants
const ( const (
metaMtime = "Mtime" // the meta key to store mtime in - eg X-Amz-Meta-Mtime metaMtime = "Mtime" // the meta key to store mtime in - eg X-Amz-Meta-Mtime
listChunkSize = 1000 // number of items to read at once listChunkSize = 1000 // number of items to read at once
maxRetries = 10 // number of retries to make of operations maxRetries = 10 // number of retries to make of operations
maxSizeForCopy = 5 * 1024 * 1024 * 1024 // The maximum size of object we can COPY maxSizeForCopy = 5 * 1024 * 1024 * 1024 // The maximum size of object we can COPY
maxFileSize = 5 * 1024 * 1024 * 1024 * 1024 // largest possible upload file size
) )
// Globals // Globals
@ -666,6 +667,11 @@ func (f *Fs) Put(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.
return fs, fs.Update(in, src, options...) return fs, fs.Update(in, src, options...)
} }
// PutStream uploads to the remote path with the modTime given of indeterminate size
func (f *Fs) PutStream(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
return f.Put(in, src, options...)
}
// Check if the bucket exists // Check if the bucket exists
// //
// NB this can return incorrect results if called immediately after bucket deletion // NB this can return incorrect results if called immediately after bucket deletion
@ -976,6 +982,12 @@ func (o *Object) Update(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOptio
u.PartSize = s3manager.MinUploadPartSize u.PartSize = s3manager.MinUploadPartSize
size := src.Size() size := src.Size()
if size == -1 {
// Make parts as small as possible while still being able to upload to the
// S3 file size limit. Rounded up to nearest MB.
u.PartSize = (((maxFileSize / s3manager.MaxUploadParts) >> 20) + 1) << 20
return
}
// Adjust PartSize until the number of parts is small enough. // Adjust PartSize until the number of parts is small enough.
if size/u.PartSize >= s3manager.MaxUploadParts { if size/u.PartSize >= s3manager.MaxUploadParts {
// Calculate partition size rounded up to the nearest MB // Calculate partition size rounded up to the nearest MB
@ -1041,9 +1053,10 @@ func (o *Object) MimeType() string {
// Check the interfaces are satisfied // Check the interfaces are satisfied
var ( var (
_ fs.Fs = &Fs{} _ fs.Fs = &Fs{}
_ fs.Copier = &Fs{} _ fs.Copier = &Fs{}
_ fs.ListRer = &Fs{} _ fs.PutStreamer = &Fs{}
_ fs.Object = &Object{} _ fs.ListRer = &Fs{}
_ fs.MimeTyper = &Object{} _ fs.Object = &Object{}
_ fs.MimeTyper = &Object{}
) )