diff --git a/fs/chunksize/chunksize.go b/fs/chunksize/chunksize.go new file mode 100644 index 000000000..ab171f07d --- /dev/null +++ b/fs/chunksize/chunksize.go @@ -0,0 +1,36 @@ +// Package chunksize calculates a suitable chunk size for large uploads +package chunksize + +import ( + "github.com/rclone/rclone/fs" +) + +/* + Calculator calculates the minimum chunk size needed to fit within the maximum number of parts, rounded up to the nearest fs.Mebi + + For most backends, (chunk_size) * (concurrent_upload_routines) memory will be required so we want to use the smallest + possible chunk size that's going to allow the upload to proceed. Rounding up to the nearest fs.Mebi on the assumption + that some backends may only allow integer type parameters when specifying the chunk size. + + Returns the default chunk size if it is sufficiently large enough to support the given file size otherwise returns the + smallest chunk size necessary to allow the upload to proceed. +*/ +func Calculator(objInfo fs.ObjectInfo, maxParts int, defaultChunkSize fs.SizeSuffix) fs.SizeSuffix { + fileSize := fs.SizeSuffix(objInfo.Size()) + requiredChunks := fileSize / defaultChunkSize + if requiredChunks < fs.SizeSuffix(maxParts) || (requiredChunks == fs.SizeSuffix(maxParts) && fileSize%defaultChunkSize == 0) { + return defaultChunkSize + } + + minChunk := fileSize / fs.SizeSuffix(maxParts) + remainder := minChunk % fs.Mebi + if remainder != 0 { + minChunk += fs.Mebi - remainder + } + if fileSize/minChunk == fs.SizeSuffix(maxParts) && fileSize%fs.SizeSuffix(maxParts) != 0 { // when right on the boundary, we need to add a MiB + minChunk += fs.Mebi + } + + fs.Debugf(objInfo, "size: %v, parts: %v, default: %v, new: %v; default chunk size insufficient, returned new chunk size", fileSize, maxParts, defaultChunkSize, minChunk) + return minChunk +} diff --git a/fs/chunksize/chunksize_test.go b/fs/chunksize/chunksize_test.go new file mode 100644 index 000000000..9cdeb70d6 --- /dev/null +++ b/fs/chunksize/chunksize_test.go @@ -0,0 +1,40 @@ +package chunksize + +import ( + "testing" + "time" + + "github.com/rclone/rclone/fs" + "github.com/rclone/rclone/fs/object" +) + +func TestComputeChunkSize(t *testing.T) { + tests := map[string]struct { + fileSize fs.SizeSuffix + maxParts int + defaultChunkSize fs.SizeSuffix + expected fs.SizeSuffix + }{ + "default size returned when file size is small enough": {fileSize: 1000, maxParts: 10000, defaultChunkSize: toSizeSuffixMiB(10), expected: toSizeSuffixMiB(10)}, + "default size returned when file size is just 1 byte small enough": {fileSize: toSizeSuffixMiB(100000) - 1, maxParts: 10000, defaultChunkSize: toSizeSuffixMiB(10), expected: toSizeSuffixMiB(10)}, + "no rounding up when everything divides evenly": {fileSize: toSizeSuffixMiB(1000000), maxParts: 10000, defaultChunkSize: toSizeSuffixMiB(100), expected: toSizeSuffixMiB(100)}, + "rounding up to nearest MiB when not quite enough parts": {fileSize: toSizeSuffixMiB(1000000), maxParts: 9999, defaultChunkSize: toSizeSuffixMiB(100), expected: toSizeSuffixMiB(101)}, + "rounding up to nearest MiB when one extra byte": {fileSize: toSizeSuffixMiB(1000000) + 1, maxParts: 10000, defaultChunkSize: toSizeSuffixMiB(100), expected: toSizeSuffixMiB(101)}, + "expected MiB value when rounding sets to absolute minimum": {fileSize: toSizeSuffixMiB(1) - 1, maxParts: 1, defaultChunkSize: toSizeSuffixMiB(1), expected: toSizeSuffixMiB(1)}, + "expected MiB value when rounding to absolute min with extra": {fileSize: toSizeSuffixMiB(1) + 1, maxParts: 1, defaultChunkSize: toSizeSuffixMiB(1), expected: toSizeSuffixMiB(2)}, + } + + for name, tc := range tests { + t.Run(name, func(t *testing.T) { + src := object.NewStaticObjectInfo("mock", time.Now(), int64(tc.fileSize), true, nil, nil) + result := Calculator(src, tc.maxParts, tc.defaultChunkSize) + if result != tc.expected { + t.Fatalf("expected: %v, got: %v", tc.expected, result) + } + }) + } +} + +func toSizeSuffixMiB(size int64) fs.SizeSuffix { + return fs.SizeSuffix(size * int64(fs.Mebi)) +}