TrueCloudLab/frostfs-s3-gw
17
3
Fork 20
Code Issues 18 Pull requests 3 Projects Releases 36 Packages 1 Activity Actions

[#417] Create multipart upload using tree service

Browse source 
Signed-off-by: Denis Kirillov <denis@nspcc.ru>
Signed-off-by: Alex Vanin <alexey@nspcc.ru>
This commit is contained in:
Denis Kirillov 2022-05-23 17:34:13 +03:00 • committed by Alex Vanin
parent 24bad60048
commit 13e01164d7
6 changed files with 135 additions and 81 deletions
Download patch file Download diff file Expand all files Collapse all files

45
api/layer/multipart_upload.go
View file

@ -26,6 +26,9 @@ const (
UploadCompletedParts = "S3-Completed-Parts"
UploadPartKeyPrefix = ".upload-"
metaPrefix = "meta-"
aclPrefix = "acl-"
MaxSizeUploadsList = 1000
MaxSizePartsList = 1000
UploadMinPartNumber = 1
@ -41,12 +44,18 @@ type (
Key string
}
CreateMultipartParams struct {
Info *UploadInfoParams
Header map[string]string
TagSet map[string]string
ACLHeaders map[string]string
}
UploadPartParams struct {
Info *UploadInfoParams
PartNumber int
Size int64
Reader io.Reader
Header map[string]string
}
UploadCopyParams struct {
@ -113,6 +122,29 @@ type (
}
)
func (n *layer) CreateMultipartUpload(ctx context.Context, p *CreateMultipartParams) error {
info := &data.MultipartInfo{
UploadID: p.Info.UploadID,
Owner: n.Owner(ctx),
Created: time.Now(),
Meta: make(map[string]string, len(p.Header)+len(p.ACLHeaders)+len(p.TagSet)),
}
for key, val := range p.Header {
info.Meta[metaPrefix+key] = val
}
for key, val := range p.ACLHeaders {
info.Meta[aclPrefix+key] = val
}
for key, val := range p.TagSet {
info.Meta[tagPrefix+key] = val
}
return n.treeService.CreateMultipart(ctx, &p.Info.Bkt.CID, p.Info.Key, info)
}
func (n *layer) UploadPart(ctx context.Context, p *UploadPartParams) (*data.ObjectInfo, error) {
if p.PartNumber != 0 {
if _, err := n.GetUploadInitInfo(ctx, p.Info); err != nil {
@ -124,17 +156,13 @@ func (n *layer) UploadPart(ctx context.Context, p *UploadPartParams) (*data.Obje
return nil, errors.GetAPIError(errors.ErrEntityTooLarge)
}
if p.Header == nil {
p.Header = make(map[string]string)
}
appendUploadHeaders(p.Header, p.Info.UploadID, p.Info.Key, p.PartNumber)
header := make(map[string]string)
appendUploadHeaders(header, p.Info.UploadID, p.Info.Key, p.PartNumber)
params := &PutSystemObjectParams{
BktInfo: p.Info.Bkt,
ObjName: FormUploadPartName(p.Info.UploadID, p.Info.Key, p.PartNumber),
Metadata: p.Header,
Prefix: "",
Metadata: header,
Reader: p.Reader,
Size: p.Size,
}
@ -609,5 +637,4 @@ func uploadInfoFromMeta(meta *object.Object, prefix, delimiter string) *UploadIn
func appendUploadHeaders(metadata map[string]string, uploadID, key string, partNumber int) {
metadata[UploadIDAttributeName] = uploadID
metadata[UploadPartNumberAttributeName] = strconv.Itoa(partNumber)
metadata[UploadKeyAttributeName] = key
}