From f2e7a2e794c18af2caa314b8fdbf6e9a4c36b8c9 Mon Sep 17 00:00:00 2001
From: Derek Battams <derek@battams.ca>
Date: Fri, 6 May 2022 15:24:27 -0400
Subject: [PATCH] chunksize: initial implementation of chunksize helper lib
---
fs/chunksize/chunksize.go | 36 ++++++++++++++++++++++++++++++
fs/chunksize/chunksize_test.go | 40 ++++++++++++++++++++++++++++++++++
2 files changed, 76 insertions(+)
create mode 100644 fs/chunksize/chunksize.go
create mode 100644 fs/chunksize/chunksize_test.go
diff --git a/fs/chunksize/chunksize.go b/fs/chunksize/chunksize.go
new file mode 100644
index 000000000..ab171f07d
--- /dev/null
+++ b/fs/chunksize/chunksize.go
@@ -0,0 +1,36 @@
+// Package chunksize calculates a suitable chunk size for large uploads
+package chunksize
+
+import (
+ "github.com/rclone/rclone/fs"
+)
+
+/*
+ Calculator calculates the minimum chunk size needed to fit within the maximum number of parts, rounded up to the nearest fs.Mebi
+
+ For most backends, (chunk_size) * (concurrent_upload_routines) memory will be required so we want to use the smallest
+ possible chunk size that's going to allow the upload to proceed. Rounding up to the nearest fs.Mebi on the assumption
+ that some backends may only allow integer type parameters when specifying the chunk size.
+
+ Returns the default chunk size if it is sufficiently large enough to support the given file size otherwise returns the
+ smallest chunk size necessary to allow the upload to proceed.
+*/
+func Calculator(objInfo fs.ObjectInfo, maxParts int, defaultChunkSize fs.SizeSuffix) fs.SizeSuffix {
+ fileSize := fs.SizeSuffix(objInfo.Size())
+ requiredChunks := fileSize / defaultChunkSize
+ if requiredChunks < fs.SizeSuffix(maxParts) || (requiredChunks == fs.SizeSuffix(maxParts) && fileSize%defaultChunkSize == 0) {
+ return defaultChunkSize
+ }
+
+ minChunk := fileSize / fs.SizeSuffix(maxParts)
+ remainder := minChunk % fs.Mebi
+ if remainder != 0 {
+ minChunk += fs.Mebi - remainder
+ }
+ if fileSize/minChunk == fs.SizeSuffix(maxParts) && fileSize%fs.SizeSuffix(maxParts) != 0 { // when right on the boundary, we need to add a MiB
+ minChunk += fs.Mebi
+ }
+
+ fs.Debugf(objInfo, "size: %v, parts: %v, default: %v, new: %v; default chunk size insufficient, returned new chunk size", fileSize, maxParts, defaultChunkSize, minChunk)
+ return minChunk
+}
diff --git a/fs/chunksize/chunksize_test.go b/fs/chunksize/chunksize_test.go
new file mode 100644
index 000000000..9cdeb70d6
--- /dev/null
+++ b/fs/chunksize/chunksize_test.go
@@ -0,0 +1,40 @@
+package chunksize
+
+import (
+ "testing"
+ "time"
+
+ "github.com/rclone/rclone/fs"
+ "github.com/rclone/rclone/fs/object"
+)
+
+func TestComputeChunkSize(t *testing.T) {
+ tests := map[string]struct {
+ fileSize fs.SizeSuffix
+ maxParts int
+ defaultChunkSize fs.SizeSuffix
+ expected fs.SizeSuffix
+ }{
+ "default size returned when file size is small enough": {fileSize: 1000, maxParts: 10000, defaultChunkSize: toSizeSuffixMiB(10), expected: toSizeSuffixMiB(10)},
+ "default size returned when file size is just 1 byte small enough": {fileSize: toSizeSuffixMiB(100000) - 1, maxParts: 10000, defaultChunkSize: toSizeSuffixMiB(10), expected: toSizeSuffixMiB(10)},
+ "no rounding up when everything divides evenly": {fileSize: toSizeSuffixMiB(1000000), maxParts: 10000, defaultChunkSize: toSizeSuffixMiB(100), expected: toSizeSuffixMiB(100)},
+ "rounding up to nearest MiB when not quite enough parts": {fileSize: toSizeSuffixMiB(1000000), maxParts: 9999, defaultChunkSize: toSizeSuffixMiB(100), expected: toSizeSuffixMiB(101)},
+ "rounding up to nearest MiB when one extra byte": {fileSize: toSizeSuffixMiB(1000000) + 1, maxParts: 10000, defaultChunkSize: toSizeSuffixMiB(100), expected: toSizeSuffixMiB(101)},
+ "expected MiB value when rounding sets to absolute minimum": {fileSize: toSizeSuffixMiB(1) - 1, maxParts: 1, defaultChunkSize: toSizeSuffixMiB(1), expected: toSizeSuffixMiB(1)},
+ "expected MiB value when rounding to absolute min with extra": {fileSize: toSizeSuffixMiB(1) + 1, maxParts: 1, defaultChunkSize: toSizeSuffixMiB(1), expected: toSizeSuffixMiB(2)},
+ }
+
+ for name, tc := range tests {
+ t.Run(name, func(t *testing.T) {
+ src := object.NewStaticObjectInfo("mock", time.Now(), int64(tc.fileSize), true, nil, nil)
+ result := Calculator(src, tc.maxParts, tc.defaultChunkSize)
+ if result != tc.expected {
+ t.Fatalf("expected: %v, got: %v", tc.expected, result)
+ }
+ })
+ }
+}
+
+func toSizeSuffixMiB(size int64) fs.SizeSuffix {
+ return fs.SizeSuffix(size * int64(fs.Mebi))
+}