r.loginov/distribution
1
0
Fork 0
forked from TrueCloudLab/distribution
Code Pull requests Activity

fix: remove disabling of multipart combine small parts (#4193)

Browse source
This commit is contained in:
Wang Yan 2023-12-19 16:10:19 +08:00 committed by GitHub
commit 4a360f9da2
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
3 changed files with 20 additions and 48 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
docs/content/storage-drivers/s3.md
View file

@ -77,6 +77,9 @@ Amazon S3 or S3 compatible services for object storage.
`loglevel`: (optional) Valid values are: `off` (default), `debug`, `debugwithsigning`, `debugwithhttpbody`, `debugwithrequestretries`, `debugwithrequesterrors` and `debugwitheventstreambody`. See the [AWS SDK for Go API reference](https://docs.aws.amazon.com/sdk-for-go/api/aws/#LogLevelType) for details.
**NOTE:** Currently the S3 storage driver only supports S3 API compatible storage that
allows parts of a multipart upload to vary in size. [Cloudflare R2 is not supported.](https://developers.cloudflare.com/r2/objects/multipart-objects/#limitations)
## S3 permission scopes
The following AWS policy is required by the registry for push and pull. Make sure to replace `S3_BUCKET_NAME` with the name of your bucket.

23
registry/storage/driver/s3-aws/s3.go
View file

@ -111,7 +111,6 @@ type DriverParameters struct {
MultipartCopyChunkSize int64
MultipartCopyMaxConcurrency int64
MultipartCopyThresholdSize int64
MultipartCombineSmallPart bool
RootDirectory string
StorageClass string
UserAgent string
@ -165,7 +164,6 @@ type driver struct {
MultipartCopyChunkSize int64
MultipartCopyMaxConcurrency int64
MultipartCopyThresholdSize int64
MultipartCombineSmallPart bool
RootDirectory string
StorageClass string
ObjectACL string
@ -405,23 +403,6 @@ func FromParameters(ctx context.Context, parameters map[string]interface{}) (*Dr
return nil, fmt.Errorf("the useDualStack parameter should be a boolean")
}
mutlipartCombineSmallPart := true
combine := parameters["multipartcombinesmallpart"]
switch combine := combine.(type) {
case string:
b, err := strconv.ParseBool(combine)
if err != nil {
return nil, fmt.Errorf("the multipartcombinesmallpart parameter should be a boolean")
}
mutlipartCombineSmallPart = b
case bool:
mutlipartCombineSmallPart = combine
case nil:
// do nothing
default:
return nil, fmt.Errorf("the multipartcombinesmallpart parameter should be a boolean")
}
sessionToken := ""
accelerateBool := false
@ -457,7 +438,6 @@ func FromParameters(ctx context.Context, parameters map[string]interface{}) (*Dr
multipartCopyChunkSize,
multipartCopyMaxConcurrency,
multipartCopyThresholdSize,
mutlipartCombineSmallPart,
fmt.Sprint(rootDirectory),
storageClass,
fmt.Sprint(userAgent),
@ -607,7 +587,6 @@ func New(ctx context.Context, params DriverParameters) (*Driver, error) {
MultipartCopyChunkSize: params.MultipartCopyChunkSize,
MultipartCopyMaxConcurrency: params.MultipartCopyMaxConcurrency,
MultipartCopyThresholdSize: params.MultipartCopyThresholdSize,
MultipartCombineSmallPart: params.MultipartCombineSmallPart,
RootDirectory: params.RootDirectory,
StorageClass: params.StorageClass,
ObjectACL: params.ObjectACL,
@ -1635,7 +1614,7 @@ func (w *writer) flush() error {
}
buf := bytes.NewBuffer(w.ready.data)
if w.driver.MultipartCombineSmallPart && (w.pending.Len() > 0 && w.pending.Len() < int(w.driver.ChunkSize)) {
if w.pending.Len() > 0 && w.pending.Len() < int(w.driver.ChunkSize) {
if _, err := buf.Write(w.pending.data); err != nil {
return err
}

42
registry/storage/driver/s3-aws/s3_test.go
View file

@ -29,23 +29,22 @@ var (
func init() {
var (
accessKey = os.Getenv("AWS_ACCESS_KEY")
secretKey = os.Getenv("AWS_SECRET_KEY")
bucket = os.Getenv("S3_BUCKET")
encrypt = os.Getenv("S3_ENCRYPT")
keyID = os.Getenv("S3_KEY_ID")
secure = os.Getenv("S3_SECURE")
skipVerify = os.Getenv("S3_SKIP_VERIFY")
v4Auth = os.Getenv("S3_V4_AUTH")
region = os.Getenv("AWS_REGION")
objectACL = os.Getenv("S3_OBJECT_ACL")
regionEndpoint = os.Getenv("REGION_ENDPOINT")
forcePathStyle = os.Getenv("AWS_S3_FORCE_PATH_STYLE")
sessionToken = os.Getenv("AWS_SESSION_TOKEN")
useDualStack = os.Getenv("S3_USE_DUALSTACK")
combineSmallPart = os.Getenv("MULTIPART_COMBINE_SMALL_PART")
accelerate = os.Getenv("S3_ACCELERATE")
logLevel = os.Getenv("S3_LOGLEVEL")
accessKey = os.Getenv("AWS_ACCESS_KEY")
secretKey = os.Getenv("AWS_SECRET_KEY")
bucket = os.Getenv("S3_BUCKET")
encrypt = os.Getenv("S3_ENCRYPT")
keyID = os.Getenv("S3_KEY_ID")
secure = os.Getenv("S3_SECURE")
skipVerify = os.Getenv("S3_SKIP_VERIFY")
v4Auth = os.Getenv("S3_V4_AUTH")
region = os.Getenv("AWS_REGION")
objectACL = os.Getenv("S3_OBJECT_ACL")
regionEndpoint = os.Getenv("REGION_ENDPOINT")
forcePathStyle = os.Getenv("AWS_S3_FORCE_PATH_STYLE")
sessionToken = os.Getenv("AWS_SESSION_TOKEN")
useDualStack = os.Getenv("S3_USE_DUALSTACK")
accelerate = os.Getenv("S3_ACCELERATE")
logLevel = os.Getenv("S3_LOGLEVEL")
)
var err error
@ -94,14 +93,6 @@ func init() {
useDualStackBool, err = strconv.ParseBool(useDualStack)
}
multipartCombineSmallPart := true
if combineSmallPart != "" {
multipartCombineSmallPart, err = strconv.ParseBool(combineSmallPart)
if err != nil {
return nil, err
}
}
accelerateBool := true
if accelerate != "" {
accelerateBool, err = strconv.ParseBool(accelerate)
@ -126,7 +117,6 @@ func init() {
defaultMultipartCopyChunkSize,
defaultMultipartCopyMaxConcurrency,
defaultMultipartCopyThresholdSize,
multipartCombineSmallPart,
rootDirectory,
storageClass,
driverName + "-test",