From 7f8d306c9c33b20d6fe4104a18facf606e0c4012 Mon Sep 17 00:00:00 2001
From: Stefan Breunig <stefan-github@yrden.de>
Date: Fri, 15 Sep 2017 20:20:32 +0200
Subject: [PATCH] s3: allow streaming upload of files with unknown file size
 (see #1614)

---
 s3/s3.go | 31 ++++++++++++++++++++++---------
 1 file changed, 22 insertions(+), 9 deletions(-)

diff --git a/s3/s3.go b/s3/s3.go
index 98f059183..7df10b18d 100644
--- a/s3/s3.go
+++ b/s3/s3.go
@@ -219,10 +219,11 @@ func init() {
 
 // Constants
 const (
-	metaMtime      = "Mtime"                // the meta key to store mtime in - eg X-Amz-Meta-Mtime
-	listChunkSize  = 1000                   // number of items to read at once
-	maxRetries     = 10                     // number of retries to make of operations
-	maxSizeForCopy = 5 * 1024 * 1024 * 1024 // The maximum size of object we can COPY
+	metaMtime      = "Mtime"                       // the meta key to store mtime in - eg X-Amz-Meta-Mtime
+	listChunkSize  = 1000                          // number of items to read at once
+	maxRetries     = 10                            // number of retries to make of operations
+	maxSizeForCopy = 5 * 1024 * 1024 * 1024        // The maximum size of object we can COPY
+	maxFileSize    = 5 * 1024 * 1024 * 1024 * 1024 // largest possible upload file size
 )
 
 // Globals
@@ -666,6 +667,11 @@ func (f *Fs) Put(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.
 	return fs, fs.Update(in, src, options...)
 }
 
+// PutStream uploads to the remote path with the modTime given of indeterminate size
+func (f *Fs) PutStream(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
+	return f.Put(in, src, options...)
+}
+
 // Check if the bucket exists
 //
 // NB this can return incorrect results if called immediately after bucket deletion
@@ -976,6 +982,12 @@ func (o *Object) Update(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOptio
 		u.PartSize = s3manager.MinUploadPartSize
 		size := src.Size()
 
+		if size == -1 {
+			// Make parts as small as possible while still being able to upload to the
+			// S3 file size limit. Rounded up to nearest MB.
+			u.PartSize = (((maxFileSize / s3manager.MaxUploadParts) >> 20) + 1) << 20
+			return
+		}
 		// Adjust PartSize until the number of parts is small enough.
 		if size/u.PartSize >= s3manager.MaxUploadParts {
 			// Calculate partition size rounded up to the nearest MB
@@ -1041,9 +1053,10 @@ func (o *Object) MimeType() string {
 
 // Check the interfaces are satisfied
 var (
-	_ fs.Fs        = &Fs{}
-	_ fs.Copier    = &Fs{}
-	_ fs.ListRer   = &Fs{}
-	_ fs.Object    = &Object{}
-	_ fs.MimeTyper = &Object{}
+	_ fs.Fs          = &Fs{}
+	_ fs.Copier      = &Fs{}
+	_ fs.PutStreamer = &Fs{}
+	_ fs.ListRer     = &Fs{}
+	_ fs.Object      = &Object{}
+	_ fs.MimeTyper   = &Object{}
 )