From 987185b9e164d0a55a8e651206fceb367b2993ed Mon Sep 17 00:00:00 2001 From: Denis Kirillov Date: Tue, 17 Aug 2021 14:57:24 +0300 Subject: [PATCH] [#196] Add bucket tagging Signed-off-by: Denis Kirillov --- api/errors/errors.go | 36 ++++++--- api/handler/not_support.go | 4 - api/handler/put.go | 35 +++++++++ api/handler/tagging.go | 142 ++++++++++++++++++++++++----------- api/handler/tagging_test.go | 1 - api/handler/unimplemented.go | 8 -- api/headers.go | 1 + api/layer/layer.go | 68 +++++++++++++++-- api/layer/util.go | 4 + docs/aws_s3_compat.md | 6 +- 10 files changed, 229 insertions(+), 76 deletions(-) diff --git a/api/errors/errors.go b/api/errors/errors.go index 104b0d3..aeda1ca 100644 --- a/api/errors/errors.go +++ b/api/errors/errors.go @@ -67,7 +67,10 @@ const ( ErrNoSuchUpload ErrNoSuchVersion ErrInvalidVersion - ErrInvalidTag + ErrInvalidArgument + ErrInvalidTagKey + ErrInvalidTagValue + ErrInvalidTagsSizeExceed ErrNotImplemented ErrPreconditionFailed ErrNotModified @@ -300,7 +303,6 @@ const ( ErrEvaluatorInvalidTimestampFormatPatternSymbol ErrEvaluatorBindingDoesNotExist ErrMissingHeaders - ErrInvalidArgument ErrInvalidColumnIndex ErrAdminConfigNotificationTargetsFailed @@ -537,10 +539,28 @@ var errorCodes = errorCodeMap{ Description: "Invalid version id specified", HTTPStatusCode: http.StatusBadRequest, }, - ErrInvalidTag: { - ErrCode: ErrInvalidTag, + ErrInvalidArgument: { + ErrCode: ErrInvalidArgument, + Code: "InvalidArgument", + Description: "The specified argument was invalid", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrInvalidTagKey: { + ErrCode: ErrInvalidTagKey, Code: "InvalidTag", - Description: "You have passed bad tag input - duplicate keys, key/values are too long, system tags were sent.", + Description: "The TagValue you have provided is invalid", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrInvalidTagValue: { + ErrCode: ErrInvalidTagValue, + Code: "InvalidTag", + Description: "The TagKey you have provided is invalid", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrInvalidTagsSizeExceed: { + ErrCode: ErrInvalidTagsSizeExceed, + Code: "BadRequest", + Description: "Object tags cannot be greater than 10", HTTPStatusCode: http.StatusBadRequest, }, ErrNotImplemented: { @@ -1853,12 +1873,6 @@ var errorCodes = errorCodeMap{ Description: "Some headers in the query are missing from the file. Check the file and try again.", HTTPStatusCode: http.StatusBadRequest, }, - ErrInvalidArgument: { - ErrCode: ErrInvalidArgument, - Code: "InvalidArgument", - Description: "The specified argument was invalid.", - HTTPStatusCode: http.StatusBadRequest, - }, ErrInvalidColumnIndex: { ErrCode: ErrInvalidColumnIndex, Code: "InvalidColumnIndex", diff --git a/api/handler/not_support.go b/api/handler/not_support.go index 019fe5b..201488d 100644 --- a/api/handler/not_support.go +++ b/api/handler/not_support.go @@ -19,10 +19,6 @@ func (h *handler) DeleteBucketEncryptionHandler(w http.ResponseWriter, r *http.R h.logAndSendError(w, "not supported", api.GetReqInfo(r.Context()), errors.GetAPIError(errors.ErrNotSupported)) } -func (h *handler) PutBucketTaggingHandler(w http.ResponseWriter, r *http.Request) { - h.logAndSendError(w, "not supported", api.GetReqInfo(r.Context()), errors.GetAPIError(errors.ErrNotSupported)) -} - func (h *handler) PutBucketNotificationHandler(w http.ResponseWriter, r *http.Request) { h.logAndSendError(w, "not supported", api.GetReqInfo(r.Context()), errors.GetAPIError(errors.ErrNotSupported)) } diff --git a/api/handler/put.go b/api/handler/put.go index 593e4da..36f1c8d 100644 --- a/api/handler/put.go +++ b/api/handler/put.go @@ -4,6 +4,7 @@ import ( "encoding/xml" "net" "net/http" + "net/url" "strings" "github.com/nspcc-dev/neofs-api-go/pkg/acl/eacl" @@ -31,6 +32,11 @@ type createBucketParams struct { func (h *handler) PutObjectHandler(w http.ResponseWriter, r *http.Request) { var newEaclTable *eacl.Table reqInfo := api.GetReqInfo(r.Context()) + tagSet, err := parseTaggingHeader(r.Header) + if err != nil { + h.logAndSendError(w, "could not parse tagging header", reqInfo, err) + return + } if containsACLHeaders(r) { objectACL, err := parseACLHeaders(r) @@ -102,6 +108,13 @@ func (h *handler) PutObjectHandler(w http.ResponseWriter, r *http.Request) { return } + if tagSet != nil { + if err = h.obj.PutObjectTagging(r.Context(), &layer.PutTaggingParams{ObjectInfo: info, TagSet: tagSet}); err != nil { + h.logAndSendError(w, "could not upload object tagging", reqInfo, err) + return + } + } + if newEaclTable != nil { p := &layer.PutBucketACLParams{ Name: reqInfo.BucketName, @@ -129,6 +142,28 @@ func containsACLHeaders(r *http.Request) bool { r.Header.Get(api.AmzGrantFullControl) != "" || r.Header.Get(api.AmzGrantWrite) != "" } +func parseTaggingHeader(header http.Header) (map[string]string, error) { + var tagSet map[string]string + if tagging := header.Get(api.AmzTagging); len(tagging) > 0 { + queries, err := url.ParseQuery(tagging) + if err != nil { + return nil, errors.GetAPIError(errors.ErrInvalidArgument) + } + if len(queries) > maxTags { + return nil, errors.GetAPIError(errors.ErrInvalidTagsSizeExceed) + } + tagSet = make(map[string]string, len(queries)) + for k, v := range queries { + tag := Tag{Key: k, Value: v[0]} + if err = checkTag(tag); err != nil { + return nil, err + } + tagSet[tag.Key] = tag.Value + } + } + return tagSet, nil +} + func parseMetadata(r *http.Request) map[string]string { res := make(map[string]string) for k, v := range r.Header { diff --git a/api/handler/tagging.go b/api/handler/tagging.go index 7268973..7ba84fc 100644 --- a/api/handler/tagging.go +++ b/api/handler/tagging.go @@ -2,7 +2,9 @@ package handler import ( "encoding/xml" + "io" "net/http" + "sort" "strings" "unicode" @@ -14,6 +16,7 @@ import ( const ( allowedTagChars = "+-=._:/@" + maxTags = 10 keyTagMaxLength = 128 valueTagMaxLength = 256 ) @@ -21,14 +24,9 @@ const ( func (h *handler) PutObjectTaggingHandler(w http.ResponseWriter, r *http.Request) { reqInfo := api.GetReqInfo(r.Context()) - tagging := new(Tagging) - if err := xml.NewDecoder(r.Body).Decode(tagging); err != nil { - h.logAndSendError(w, "could not decode body", reqInfo, errors.GetAPIError(errors.ErrMalformedXML)) - return - } - - if err := checkTagSet(tagging.TagSet); err != nil { - h.logAndSendError(w, "some tags are invalid", reqInfo, err) + tagSet, err := readTagSet(r.Body) + if err != nil { + h.logAndSendError(w, "could not read tag set", reqInfo, err) return } @@ -44,11 +42,6 @@ func (h *handler) PutObjectTaggingHandler(w http.ResponseWriter, r *http.Request return } - tagSet := make(map[string]string, len(tagging.TagSet)) - for _, tag := range tagging.TagSet { - tagSet[tag.Key] = tag.Value - } - p2 := &layer.PutTaggingParams{ ObjectInfo: objInfo, TagSet: tagSet, @@ -81,31 +74,12 @@ func (h *handler) GetObjectTaggingHandler(w http.ResponseWriter, r *http.Request return } - tagging := &Tagging{} - for k, v := range tagSet { - tagging.TagSet = append(tagging.TagSet, Tag{Key: k, Value: v}) - } - w.Header().Set(api.AmzVersionID, objInfo.Version()) - if err = api.EncodeToResponse(w, tagging); err != nil { + if err = api.EncodeToResponse(w, encodeTagging(tagSet)); err != nil { h.logAndSendError(w, "something went wrong", reqInfo, err) } } -func checkTagSet(tagSet []Tag) error { - if len(tagSet) > 10 { - return errors.GetAPIError(errors.ErrInvalidTag) - } - - for _, tag := range tagSet { - if err := checkTag(tag); err != nil { - return err - } - } - - return nil -} - func (h *handler) DeleteObjectTaggingHandler(w http.ResponseWriter, r *http.Request) { reqInfo := api.GetReqInfo(r.Context()) @@ -128,34 +102,114 @@ func (h *handler) DeleteObjectTaggingHandler(w http.ResponseWriter, r *http.Requ w.WriteHeader(http.StatusNoContent) } +func (h *handler) PutBucketTaggingHandler(w http.ResponseWriter, r *http.Request) { + reqInfo := api.GetReqInfo(r.Context()) + + tagSet, err := readTagSet(r.Body) + if err != nil { + h.logAndSendError(w, "could not read tag set", reqInfo, err) + return + } + + if err := h.obj.PutBucketTagging(r.Context(), reqInfo.BucketName, tagSet); err != nil { + h.logAndSendError(w, "could not put object tagging", reqInfo, err) + } +} + +func (h *handler) GetBucketTaggingHandler(w http.ResponseWriter, r *http.Request) { + reqInfo := api.GetReqInfo(r.Context()) + + tagSet, err := h.obj.GetBucketTagging(r.Context(), reqInfo.BucketName) + if err != nil { + h.logAndSendError(w, "could not get object tagging", reqInfo, err) + return + } + + if err = api.EncodeToResponse(w, encodeTagging(tagSet)); err != nil { + h.logAndSendError(w, "something went wrong", reqInfo, err) + } +} + +func (h *handler) DeleteBucketTaggingHandler(w http.ResponseWriter, r *http.Request) { + reqInfo := api.GetReqInfo(r.Context()) + if err := h.obj.DeleteBucketTagging(r.Context(), reqInfo.BucketName); err != nil { + h.logAndSendError(w, "could not delete object tagging", reqInfo, err) + } + w.WriteHeader(http.StatusNoContent) +} + +func readTagSet(reader io.Reader) (map[string]string, error) { + tagging := new(Tagging) + if err := xml.NewDecoder(reader).Decode(tagging); err != nil { + return nil, errors.GetAPIError(errors.ErrMalformedXML) + } + + if err := checkTagSet(tagging.TagSet); err != nil { + return nil, err + } + + tagSet := make(map[string]string, len(tagging.TagSet)) + for _, tag := range tagging.TagSet { + tagSet[tag.Key] = tag.Value + } + + return tagSet, nil +} + +func encodeTagging(tagSet map[string]string) *Tagging { + tagging := &Tagging{} + for k, v := range tagSet { + tagging.TagSet = append(tagging.TagSet, Tag{Key: k, Value: v}) + } + sort.Slice(tagging.TagSet, func(i, j int) bool { + return tagging.TagSet[i].Key < tagging.TagSet[j].Key + }) + + return tagging +} + +func checkTagSet(tagSet []Tag) error { + if len(tagSet) > maxTags { + return errors.GetAPIError(errors.ErrInvalidTagsSizeExceed) + } + + for _, tag := range tagSet { + if err := checkTag(tag); err != nil { + return err + } + } + + return nil +} + func checkTag(tag Tag) error { if len(tag.Key) < 1 || len(tag.Key) > keyTagMaxLength { - return errors.GetAPIError(errors.ErrInvalidTag) + return errors.GetAPIError(errors.ErrInvalidTagKey) } - if len(tag.Value) < 1 || len(tag.Value) > valueTagMaxLength { - return errors.GetAPIError(errors.ErrInvalidTag) + if len(tag.Value) > valueTagMaxLength { + return errors.GetAPIError(errors.ErrInvalidTagValue) } if strings.HasPrefix(tag.Key, "aws:") { - return errors.GetAPIError(errors.ErrInvalidTag) + return errors.GetAPIError(errors.ErrInvalidTagKey) } - if err := checkCharacters(tag.Key); err != nil { - return err + if !isValidTag(tag.Key) { + return errors.GetAPIError(errors.ErrInvalidTagKey) } - if err := checkCharacters(tag.Value); err != nil { - return err + if !isValidTag(tag.Value) { + return errors.GetAPIError(errors.ErrInvalidTagValue) } return nil } -func checkCharacters(str string) error { +func isValidTag(str string) bool { for _, r := range str { if !unicode.IsLetter(r) && !unicode.IsDigit(r) && !unicode.IsSpace(r) && !strings.ContainsRune(allowedTagChars, r) { - return errors.GetAPIError(errors.ErrInvalidTag) + return false } } - return nil + return true } diff --git a/api/handler/tagging_test.go b/api/handler/tagging_test.go index e305df9..98b39a7 100644 --- a/api/handler/tagging_test.go +++ b/api/handler/tagging_test.go @@ -23,7 +23,6 @@ func TestTagsValidity(t *testing.T) { }{ {tag: Tag{}, valid: false}, {tag: Tag{Key: "", Value: "1"}, valid: false}, - {tag: Tag{Key: "2", Value: ""}, valid: false}, {tag: Tag{Key: "aws:key", Value: "val"}, valid: false}, {tag: Tag{Key: "key~", Value: "val"}, valid: false}, {tag: Tag{Key: "key\\", Value: "val"}, valid: false}, diff --git a/api/handler/unimplemented.go b/api/handler/unimplemented.go index 61cbe36..4567129 100644 --- a/api/handler/unimplemented.go +++ b/api/handler/unimplemented.go @@ -83,18 +83,10 @@ func (h *handler) GetBucketReplicationHandler(w http.ResponseWriter, r *http.Req h.logAndSendError(w, "not implemented", api.GetReqInfo(r.Context()), errors.GetAPIError(errors.ErrNotImplemented)) } -func (h *handler) GetBucketTaggingHandler(w http.ResponseWriter, r *http.Request) { - h.logAndSendError(w, "not implemented", api.GetReqInfo(r.Context()), errors.GetAPIError(errors.ErrNotImplemented)) -} - func (h *handler) DeleteBucketWebsiteHandler(w http.ResponseWriter, r *http.Request) { h.logAndSendError(w, "not implemented", api.GetReqInfo(r.Context()), errors.GetAPIError(errors.ErrNotImplemented)) } -func (h *handler) DeleteBucketTaggingHandler(w http.ResponseWriter, r *http.Request) { - h.logAndSendError(w, "not implemented", api.GetReqInfo(r.Context()), errors.GetAPIError(errors.ErrNotImplemented)) -} - func (h *handler) GetBucketObjectLockConfigHandler(w http.ResponseWriter, r *http.Request) { h.logAndSendError(w, "not implemented", api.GetReqInfo(r.Context()), errors.GetAPIError(errors.ErrNotImplemented)) } diff --git a/api/headers.go b/api/headers.go index a5b9e9f..0191100 100644 --- a/api/headers.go +++ b/api/headers.go @@ -6,6 +6,7 @@ const ( AmzMetadataDirective = "X-Amz-Metadata-Directive" AmzVersionID = "X-Amz-Version-Id" AmzTaggingCount = "X-Amz-Tagging-Count" + AmzTagging = "X-Amz-Tagging" LastModified = "Last-Modified" Date = "Date" diff --git a/api/layer/layer.go b/api/layer/layer.go index 88b66d0..9463075 100644 --- a/api/layer/layer.go +++ b/api/layer/layer.go @@ -164,9 +164,11 @@ type ( GetObject(ctx context.Context, p *GetObjectParams) error GetObjectInfo(ctx context.Context, p *HeadObjectParams) (*ObjectInfo, error) GetObjectTagging(ctx context.Context, p *ObjectInfo) (map[string]string, error) + GetBucketTagging(ctx context.Context, bucket string) (map[string]string, error) PutObject(ctx context.Context, p *PutObjectParams) (*ObjectInfo, error) PutObjectTagging(ctx context.Context, p *PutTaggingParams) error + PutBucketTagging(ctx context.Context, bucket string, tagSet map[string]string) error CopyObject(ctx context.Context, p *CopyObjectParams) (*ObjectInfo, error) @@ -176,10 +178,14 @@ type ( DeleteObjects(ctx context.Context, bucket string, objects []*VersionedObject) []error DeleteObjectTagging(ctx context.Context, p *ObjectInfo) error + DeleteBucketTagging(ctx context.Context, bucket string) error } ) -const tagPrefix = "S3-Tag-" +const ( + tagPrefix = "S3-Tag-" + tagEmptyMark = "\\" +) func (t *VersionedObject) String() string { return t.Name + ":" + t.VersionID @@ -387,17 +393,38 @@ func (n *layer) GetObjectTagging(ctx context.Context, oi *ObjectInfo) (map[strin return nil, err } + return formTagSet(objInfo), nil +} + +// GetBucketTagging from storage. +func (n *layer) GetBucketTagging(ctx context.Context, bucketName string) (map[string]string, error) { + bktInfo, err := n.GetBucketInfo(ctx, bucketName) + if err != nil { + return nil, err + } + + objInfo, err := n.getSystemObject(ctx, bktInfo, formBucketTagObjectName(bucketName)) + if err != nil && !errors.IsS3Error(err, errors.ErrNoSuchKey) { + return nil, err + } + + return formTagSet(objInfo), nil +} + +func formTagSet(objInfo *ObjectInfo) map[string]string { var tagSet map[string]string if objInfo != nil { tagSet = make(map[string]string, len(objInfo.Headers)) for k, v := range objInfo.Headers { if strings.HasPrefix(k, tagPrefix) { + if v == tagEmptyMark { + v = "" + } tagSet[strings.TrimPrefix(k, tagPrefix)] = v } } } - - return tagSet, nil + return tagSet } // PutObjectTagging into storage. @@ -415,9 +442,27 @@ func (n *layer) PutObjectTagging(ctx context.Context, p *PutTaggingParams) error return nil } +// PutBucketTagging into storage. +func (n *layer) PutBucketTagging(ctx context.Context, bucketName string, tagSet map[string]string) error { + bktInfo, err := n.GetBucketInfo(ctx, bucketName) + if err != nil { + return err + } + + if _, err = n.putSystemObject(ctx, bktInfo, formBucketTagObjectName(bucketName), tagSet, tagPrefix); err != nil { + return err + } + + return nil +} + // DeleteObjectTagging from storage. func (n *layer) DeleteObjectTagging(ctx context.Context, p *ObjectInfo) error { - oid, err := n.objectFindID(ctx, &findParams{cid: p.CID(), attr: objectSystemAttributeName, val: p.TagsObject()}) + return n.deleteSystemObject(ctx, p.CID(), p.TagsObject()) +} + +func (n *layer) deleteSystemObject(ctx context.Context, bktCID *cid.ID, name string) error { + oid, err := n.objectFindID(ctx, &findParams{cid: bktCID, attr: objectSystemAttributeName, val: name}) if err != nil { if errors.IsS3Error(err, errors.ErrNoSuchKey) { return nil @@ -425,7 +470,17 @@ func (n *layer) DeleteObjectTagging(ctx context.Context, p *ObjectInfo) error { return err } - return n.objectDelete(ctx, p.CID(), oid) + return n.objectDelete(ctx, bktCID, oid) +} + +// DeleteBucketTagging from storage. +func (n *layer) DeleteBucketTagging(ctx context.Context, bucketName string) error { + bktInfo, err := n.GetBucketInfo(ctx, bucketName) + if err != nil { + return err + } + + return n.deleteSystemObject(ctx, bktInfo.CID, formBucketTagObjectName(bucketName)) } func (n *layer) putSystemObject(ctx context.Context, bktInfo *cache.BucketInfo, objName string, metadata map[string]string, prefix string) (*object.ID, error) { @@ -453,6 +508,9 @@ func (n *layer) putSystemObject(ctx context.Context, bktInfo *cache.BucketInfo, for k, v := range metadata { attr := object.NewAttribute() attr.SetKey(prefix + k) + if prefix == tagPrefix && v == "" { + v = tagEmptyMark + } attr.SetValue(v) attributes = append(attributes, attr) } diff --git a/api/layer/util.go b/api/layer/util.go index 7b5ad5d..abcd9e6 100644 --- a/api/layer/util.go +++ b/api/layer/util.go @@ -198,3 +198,7 @@ func GetBoxData(ctx context.Context) (*accessbox.Box, error) { } return boxData, nil } + +func formBucketTagObjectName(name string) string { + return ".tagset." + name +} diff --git a/docs/aws_s3_compat.md b/docs/aws_s3_compat.md index 5ba5094..899ae7f 100644 --- a/docs/aws_s3_compat.md +++ b/docs/aws_s3_compat.md @@ -209,9 +209,9 @@ See also `GetObject` and other method parameters. | | Method | Comments | |----|---------------------|----------| -| 🔴 | DeleteBucketTagging | | -| 🔴 | GetBucketTagging | | -| 🔴 | PutBucketTagging | | +| 🟢 | DeleteBucketTagging | | +| 🟢 | GetBucketTagging | | +| 🟢 | PutBucketTagging | | ## Tiering