Merge pull request #1267 from harshavardhana/possible-fix-memory

Implement Size() and Len() to know the optimal size.

internal/backend/s3/s3.go

@@ -4,7 +4,6 @@ import (
 	"context"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"os"
 	"path"
 	"strings"
@@ -198,6 +197,53 @@ func (be *Backend) Path() string {
 	return be.cfg.Prefix
 }
 
+// nopCloserFile wraps *os.File and overwrites the Close() method with method
+// that does nothing. In addition, the method Len() is implemented, which
+// returns the size of the file (filesize - current offset).
+type nopCloserFile struct {
+	*os.File
+}
+
+func (f nopCloserFile) Close() error {
+	debug.Log("prevented Close()")
+	return nil
+}
+
+// Len returns the remaining length of the file (filesize - current offset).
+func (f nopCloserFile) Len() int {
+	debug.Log("Len() called")
+	fi, err := f.Stat()
+	if err != nil {
+		panic(err)
+	}
+
+	pos, err := f.Seek(0, io.SeekCurrent)
+	if err != nil {
+		panic(err)
+	}
+
+	size := fi.Size() - pos
+	debug.Log("returning file size %v", size)
+	return int(size)
+}
+
+type lenner interface {
+	Len() int
+	io.Reader
+}
+
+// nopCloserLenner wraps a lenner and overwrites the Close() method with method
+// that does nothing. In addition, the method Size() is implemented, which
+// returns the size of the file (filesize - current offset).
+type nopCloserLenner struct {
+	lenner
+}
+
+func (f *nopCloserLenner) Close() error {
+	debug.Log("prevented Close()")
+	return nil
+}
+
 // Save stores data in the backend at the handle.
 func (be *Backend) Save(ctx context.Context, h restic.Handle, rd io.Reader) (err error) {
 	debug.Log("Save %v", h)
@@ -215,8 +261,17 @@ func (be *Backend) Save(ctx context.Context, h restic.Handle, rd io.Reader) (err
 		return errors.New("key already exists")
 	}
 
-	// prevent the HTTP client from closing a file
-	rd = ioutil.NopCloser(rd)
+	// FIXME: This is a workaround once we move to minio-go 4.0.x this can be
+	// removed and size can be directly provided.
+	if f, ok := rd.(*os.File); ok {
+		debug.Log("reader is %#T, using nopCloserFile{}", rd)
+		rd = nopCloserFile{f}
+	} else if l, ok := rd.(lenner); ok {
+		debug.Log("reader is %#T, using nopCloserLenner{}", rd)
+		rd = nopCloserLenner{l}
+	} else {
+		debug.Log("reader is %#T, no specific workaround enabled", rd)
+	}
 
 	be.sem.GetToken()
 	debug.Log("PutObject(%v, %v)", be.cfg.Bucket, objName)