Merge pull request #3556 from huangnauh/main
feat: add option to disable combining the pending partpull/3571/head
commit
c53f110049
|
@ -97,6 +97,7 @@ type DriverParameters struct {
|
||||||
MultipartCopyChunkSize int64
|
MultipartCopyChunkSize int64
|
||||||
MultipartCopyMaxConcurrency int64
|
MultipartCopyMaxConcurrency int64
|
||||||
MultipartCopyThresholdSize int64
|
MultipartCopyThresholdSize int64
|
||||||
|
MultipartCombineSmallPart bool
|
||||||
RootDirectory string
|
RootDirectory string
|
||||||
StorageClass string
|
StorageClass string
|
||||||
UserAgent string
|
UserAgent string
|
||||||
|
@ -146,6 +147,7 @@ type driver struct {
|
||||||
MultipartCopyChunkSize int64
|
MultipartCopyChunkSize int64
|
||||||
MultipartCopyMaxConcurrency int64
|
MultipartCopyMaxConcurrency int64
|
||||||
MultipartCopyThresholdSize int64
|
MultipartCopyThresholdSize int64
|
||||||
|
MultipartCombineSmallPart bool
|
||||||
RootDirectory string
|
RootDirectory string
|
||||||
StorageClass string
|
StorageClass string
|
||||||
ObjectACL string
|
ObjectACL string
|
||||||
|
@ -356,6 +358,23 @@ func FromParameters(parameters map[string]interface{}) (*Driver, error) {
|
||||||
return nil, fmt.Errorf("the useDualStack parameter should be a boolean")
|
return nil, fmt.Errorf("the useDualStack parameter should be a boolean")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
mutlipartCombineSmallPart := true
|
||||||
|
combine := parameters["multipartcombinesmallpart"]
|
||||||
|
switch combine := combine.(type) {
|
||||||
|
case string:
|
||||||
|
b, err := strconv.ParseBool(combine)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("the multipartcombinesmallpart parameter should be a boolean")
|
||||||
|
}
|
||||||
|
mutlipartCombineSmallPart = b
|
||||||
|
case bool:
|
||||||
|
mutlipartCombineSmallPart = combine
|
||||||
|
case nil:
|
||||||
|
// do nothing
|
||||||
|
default:
|
||||||
|
return nil, fmt.Errorf("the multipartcombinesmallpart parameter should be a boolean")
|
||||||
|
}
|
||||||
|
|
||||||
sessionToken := ""
|
sessionToken := ""
|
||||||
|
|
||||||
params := DriverParameters{
|
params := DriverParameters{
|
||||||
|
@ -373,6 +392,7 @@ func FromParameters(parameters map[string]interface{}) (*Driver, error) {
|
||||||
multipartCopyChunkSize,
|
multipartCopyChunkSize,
|
||||||
multipartCopyMaxConcurrency,
|
multipartCopyMaxConcurrency,
|
||||||
multipartCopyThresholdSize,
|
multipartCopyThresholdSize,
|
||||||
|
mutlipartCombineSmallPart,
|
||||||
fmt.Sprint(rootDirectory),
|
fmt.Sprint(rootDirectory),
|
||||||
storageClass,
|
storageClass,
|
||||||
fmt.Sprint(userAgent),
|
fmt.Sprint(userAgent),
|
||||||
|
@ -497,6 +517,7 @@ func New(params DriverParameters) (*Driver, error) {
|
||||||
MultipartCopyChunkSize: params.MultipartCopyChunkSize,
|
MultipartCopyChunkSize: params.MultipartCopyChunkSize,
|
||||||
MultipartCopyMaxConcurrency: params.MultipartCopyMaxConcurrency,
|
MultipartCopyMaxConcurrency: params.MultipartCopyMaxConcurrency,
|
||||||
MultipartCopyThresholdSize: params.MultipartCopyThresholdSize,
|
MultipartCopyThresholdSize: params.MultipartCopyThresholdSize,
|
||||||
|
MultipartCombineSmallPart: params.MultipartCombineSmallPart,
|
||||||
RootDirectory: params.RootDirectory,
|
RootDirectory: params.RootDirectory,
|
||||||
StorageClass: params.StorageClass,
|
StorageClass: params.StorageClass,
|
||||||
ObjectACL: params.ObjectACL,
|
ObjectACL: params.ObjectACL,
|
||||||
|
@ -1386,7 +1407,7 @@ func (w *writer) flushPart() error {
|
||||||
// nothing to write
|
// nothing to write
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
if len(w.pendingPart) < int(w.driver.ChunkSize) {
|
if w.driver.MultipartCombineSmallPart && len(w.pendingPart) < int(w.driver.ChunkSize) {
|
||||||
// closing with a small pending part
|
// closing with a small pending part
|
||||||
// combine ready and pending to avoid writing a small part
|
// combine ready and pending to avoid writing a small part
|
||||||
w.readyPart = append(w.readyPart, w.pendingPart...)
|
w.readyPart = append(w.readyPart, w.pendingPart...)
|
||||||
|
|
|
@ -43,6 +43,7 @@ func init() {
|
||||||
regionEndpoint := os.Getenv("REGION_ENDPOINT")
|
regionEndpoint := os.Getenv("REGION_ENDPOINT")
|
||||||
sessionToken := os.Getenv("AWS_SESSION_TOKEN")
|
sessionToken := os.Getenv("AWS_SESSION_TOKEN")
|
||||||
useDualStack := os.Getenv("S3_USE_DUALSTACK")
|
useDualStack := os.Getenv("S3_USE_DUALSTACK")
|
||||||
|
combineSmallPart := os.Getenv("MULTIPART_COMBINE_SMALL_PART")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
panic(err)
|
panic(err)
|
||||||
}
|
}
|
||||||
|
@ -84,6 +85,10 @@ func init() {
|
||||||
useDualStackBool := false
|
useDualStackBool := false
|
||||||
if useDualStack != "" {
|
if useDualStack != "" {
|
||||||
useDualStackBool, err = strconv.ParseBool(useDualStack)
|
useDualStackBool, err = strconv.ParseBool(useDualStack)
|
||||||
|
}
|
||||||
|
multipartCombineSmallPart := true
|
||||||
|
if combineSmallPart != "" {
|
||||||
|
multipartCombineSmallPart, err = strconv.ParseBool(combineSmallPart)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
@ -104,6 +109,7 @@ func init() {
|
||||||
defaultMultipartCopyChunkSize,
|
defaultMultipartCopyChunkSize,
|
||||||
defaultMultipartCopyMaxConcurrency,
|
defaultMultipartCopyMaxConcurrency,
|
||||||
defaultMultipartCopyThresholdSize,
|
defaultMultipartCopyThresholdSize,
|
||||||
|
multipartCombineSmallPart,
|
||||||
rootDirectory,
|
rootDirectory,
|
||||||
storageClass,
|
storageClass,
|
||||||
driverName + "-test",
|
driverName + "-test",
|
||||||
|
|
Loading…
Reference in New Issue