Merge pull request #3556 from huangnauh/main

feat: add option to disable combining the pending part
This commit is contained in:
Milos Gajdos 2022-01-18 08:58:57 +00:00 committed by GitHub
commit c53f110049
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
2 changed files with 28 additions and 1 deletions

View file

@ -97,6 +97,7 @@ type DriverParameters struct {
MultipartCopyChunkSize int64 MultipartCopyChunkSize int64
MultipartCopyMaxConcurrency int64 MultipartCopyMaxConcurrency int64
MultipartCopyThresholdSize int64 MultipartCopyThresholdSize int64
MultipartCombineSmallPart bool
RootDirectory string RootDirectory string
StorageClass string StorageClass string
UserAgent string UserAgent string
@ -146,6 +147,7 @@ type driver struct {
MultipartCopyChunkSize int64 MultipartCopyChunkSize int64
MultipartCopyMaxConcurrency int64 MultipartCopyMaxConcurrency int64
MultipartCopyThresholdSize int64 MultipartCopyThresholdSize int64
MultipartCombineSmallPart bool
RootDirectory string RootDirectory string
StorageClass string StorageClass string
ObjectACL string ObjectACL string
@ -356,6 +358,23 @@ func FromParameters(parameters map[string]interface{}) (*Driver, error) {
return nil, fmt.Errorf("the useDualStack parameter should be a boolean") return nil, fmt.Errorf("the useDualStack parameter should be a boolean")
} }
mutlipartCombineSmallPart := true
combine := parameters["multipartcombinesmallpart"]
switch combine := combine.(type) {
case string:
b, err := strconv.ParseBool(combine)
if err != nil {
return nil, fmt.Errorf("the multipartcombinesmallpart parameter should be a boolean")
}
mutlipartCombineSmallPart = b
case bool:
mutlipartCombineSmallPart = combine
case nil:
// do nothing
default:
return nil, fmt.Errorf("the multipartcombinesmallpart parameter should be a boolean")
}
sessionToken := "" sessionToken := ""
params := DriverParameters{ params := DriverParameters{
@ -373,6 +392,7 @@ func FromParameters(parameters map[string]interface{}) (*Driver, error) {
multipartCopyChunkSize, multipartCopyChunkSize,
multipartCopyMaxConcurrency, multipartCopyMaxConcurrency,
multipartCopyThresholdSize, multipartCopyThresholdSize,
mutlipartCombineSmallPart,
fmt.Sprint(rootDirectory), fmt.Sprint(rootDirectory),
storageClass, storageClass,
fmt.Sprint(userAgent), fmt.Sprint(userAgent),
@ -497,6 +517,7 @@ func New(params DriverParameters) (*Driver, error) {
MultipartCopyChunkSize: params.MultipartCopyChunkSize, MultipartCopyChunkSize: params.MultipartCopyChunkSize,
MultipartCopyMaxConcurrency: params.MultipartCopyMaxConcurrency, MultipartCopyMaxConcurrency: params.MultipartCopyMaxConcurrency,
MultipartCopyThresholdSize: params.MultipartCopyThresholdSize, MultipartCopyThresholdSize: params.MultipartCopyThresholdSize,
MultipartCombineSmallPart: params.MultipartCombineSmallPart,
RootDirectory: params.RootDirectory, RootDirectory: params.RootDirectory,
StorageClass: params.StorageClass, StorageClass: params.StorageClass,
ObjectACL: params.ObjectACL, ObjectACL: params.ObjectACL,
@ -1386,7 +1407,7 @@ func (w *writer) flushPart() error {
// nothing to write // nothing to write
return nil return nil
} }
if len(w.pendingPart) < int(w.driver.ChunkSize) { if w.driver.MultipartCombineSmallPart && len(w.pendingPart) < int(w.driver.ChunkSize) {
// closing with a small pending part // closing with a small pending part
// combine ready and pending to avoid writing a small part // combine ready and pending to avoid writing a small part
w.readyPart = append(w.readyPart, w.pendingPart...) w.readyPart = append(w.readyPart, w.pendingPart...)

View file

@ -43,6 +43,7 @@ func init() {
regionEndpoint := os.Getenv("REGION_ENDPOINT") regionEndpoint := os.Getenv("REGION_ENDPOINT")
sessionToken := os.Getenv("AWS_SESSION_TOKEN") sessionToken := os.Getenv("AWS_SESSION_TOKEN")
useDualStack := os.Getenv("S3_USE_DUALSTACK") useDualStack := os.Getenv("S3_USE_DUALSTACK")
combineSmallPart := os.Getenv("MULTIPART_COMBINE_SMALL_PART")
if err != nil { if err != nil {
panic(err) panic(err)
} }
@ -84,6 +85,10 @@ func init() {
useDualStackBool := false useDualStackBool := false
if useDualStack != "" { if useDualStack != "" {
useDualStackBool, err = strconv.ParseBool(useDualStack) useDualStackBool, err = strconv.ParseBool(useDualStack)
}
multipartCombineSmallPart := true
if combineSmallPart != "" {
multipartCombineSmallPart, err = strconv.ParseBool(combineSmallPart)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@ -104,6 +109,7 @@ func init() {
defaultMultipartCopyChunkSize, defaultMultipartCopyChunkSize,
defaultMultipartCopyMaxConcurrency, defaultMultipartCopyMaxConcurrency,
defaultMultipartCopyThresholdSize, defaultMultipartCopyThresholdSize,
multipartCombineSmallPart,
rootDirectory, rootDirectory,
storageClass, storageClass,
driverName + "-test", driverName + "-test",