[#196] Add bucket tagging

Signed-off-by: Denis Kirillov <denis@nspcc.ru>
This commit is contained in:
Denis Kirillov 2021-08-17 14:57:24 +03:00 committed by Alex Vanin
parent 16da1aba64
commit 987185b9e1
10 changed files with 229 additions and 76 deletions

View file

@ -67,7 +67,10 @@ const (
ErrNoSuchUpload
ErrNoSuchVersion
ErrInvalidVersion
ErrInvalidTag
ErrInvalidArgument
ErrInvalidTagKey
ErrInvalidTagValue
ErrInvalidTagsSizeExceed
ErrNotImplemented
ErrPreconditionFailed
ErrNotModified
@ -300,7 +303,6 @@ const (
ErrEvaluatorInvalidTimestampFormatPatternSymbol
ErrEvaluatorBindingDoesNotExist
ErrMissingHeaders
ErrInvalidArgument
ErrInvalidColumnIndex
ErrAdminConfigNotificationTargetsFailed
@ -537,10 +539,28 @@ var errorCodes = errorCodeMap{
Description: "Invalid version id specified",
HTTPStatusCode: http.StatusBadRequest,
},
ErrInvalidTag: {
ErrCode: ErrInvalidTag,
ErrInvalidArgument: {
ErrCode: ErrInvalidArgument,
Code: "InvalidArgument",
Description: "The specified argument was invalid",
HTTPStatusCode: http.StatusBadRequest,
},
ErrInvalidTagKey: {
ErrCode: ErrInvalidTagKey,
Code: "InvalidTag",
Description: "You have passed bad tag input - duplicate keys, key/values are too long, system tags were sent.",
Description: "The TagValue you have provided is invalid",
HTTPStatusCode: http.StatusBadRequest,
},
ErrInvalidTagValue: {
ErrCode: ErrInvalidTagValue,
Code: "InvalidTag",
Description: "The TagKey you have provided is invalid",
HTTPStatusCode: http.StatusBadRequest,
},
ErrInvalidTagsSizeExceed: {
ErrCode: ErrInvalidTagsSizeExceed,
Code: "BadRequest",
Description: "Object tags cannot be greater than 10",
HTTPStatusCode: http.StatusBadRequest,
},
ErrNotImplemented: {
@ -1853,12 +1873,6 @@ var errorCodes = errorCodeMap{
Description: "Some headers in the query are missing from the file. Check the file and try again.",
HTTPStatusCode: http.StatusBadRequest,
},
ErrInvalidArgument: {
ErrCode: ErrInvalidArgument,
Code: "InvalidArgument",
Description: "The specified argument was invalid.",
HTTPStatusCode: http.StatusBadRequest,
},
ErrInvalidColumnIndex: {
ErrCode: ErrInvalidColumnIndex,
Code: "InvalidColumnIndex",

View file

@ -19,10 +19,6 @@ func (h *handler) DeleteBucketEncryptionHandler(w http.ResponseWriter, r *http.R
h.logAndSendError(w, "not supported", api.GetReqInfo(r.Context()), errors.GetAPIError(errors.ErrNotSupported))
}
func (h *handler) PutBucketTaggingHandler(w http.ResponseWriter, r *http.Request) {
h.logAndSendError(w, "not supported", api.GetReqInfo(r.Context()), errors.GetAPIError(errors.ErrNotSupported))
}
func (h *handler) PutBucketNotificationHandler(w http.ResponseWriter, r *http.Request) {
h.logAndSendError(w, "not supported", api.GetReqInfo(r.Context()), errors.GetAPIError(errors.ErrNotSupported))
}

View file

@ -4,6 +4,7 @@ import (
"encoding/xml"
"net"
"net/http"
"net/url"
"strings"
"github.com/nspcc-dev/neofs-api-go/pkg/acl/eacl"
@ -31,6 +32,11 @@ type createBucketParams struct {
func (h *handler) PutObjectHandler(w http.ResponseWriter, r *http.Request) {
var newEaclTable *eacl.Table
reqInfo := api.GetReqInfo(r.Context())
tagSet, err := parseTaggingHeader(r.Header)
if err != nil {
h.logAndSendError(w, "could not parse tagging header", reqInfo, err)
return
}
if containsACLHeaders(r) {
objectACL, err := parseACLHeaders(r)
@ -102,6 +108,13 @@ func (h *handler) PutObjectHandler(w http.ResponseWriter, r *http.Request) {
return
}
if tagSet != nil {
if err = h.obj.PutObjectTagging(r.Context(), &layer.PutTaggingParams{ObjectInfo: info, TagSet: tagSet}); err != nil {
h.logAndSendError(w, "could not upload object tagging", reqInfo, err)
return
}
}
if newEaclTable != nil {
p := &layer.PutBucketACLParams{
Name: reqInfo.BucketName,
@ -129,6 +142,28 @@ func containsACLHeaders(r *http.Request) bool {
r.Header.Get(api.AmzGrantFullControl) != "" || r.Header.Get(api.AmzGrantWrite) != ""
}
func parseTaggingHeader(header http.Header) (map[string]string, error) {
var tagSet map[string]string
if tagging := header.Get(api.AmzTagging); len(tagging) > 0 {
queries, err := url.ParseQuery(tagging)
if err != nil {
return nil, errors.GetAPIError(errors.ErrInvalidArgument)
}
if len(queries) > maxTags {
return nil, errors.GetAPIError(errors.ErrInvalidTagsSizeExceed)
}
tagSet = make(map[string]string, len(queries))
for k, v := range queries {
tag := Tag{Key: k, Value: v[0]}
if err = checkTag(tag); err != nil {
return nil, err
}
tagSet[tag.Key] = tag.Value
}
}
return tagSet, nil
}
func parseMetadata(r *http.Request) map[string]string {
res := make(map[string]string)
for k, v := range r.Header {

View file

@ -2,7 +2,9 @@ package handler
import (
"encoding/xml"
"io"
"net/http"
"sort"
"strings"
"unicode"
@ -14,6 +16,7 @@ import (
const (
allowedTagChars = "+-=._:/@"
maxTags = 10
keyTagMaxLength = 128
valueTagMaxLength = 256
)
@ -21,14 +24,9 @@ const (
func (h *handler) PutObjectTaggingHandler(w http.ResponseWriter, r *http.Request) {
reqInfo := api.GetReqInfo(r.Context())
tagging := new(Tagging)
if err := xml.NewDecoder(r.Body).Decode(tagging); err != nil {
h.logAndSendError(w, "could not decode body", reqInfo, errors.GetAPIError(errors.ErrMalformedXML))
return
}
if err := checkTagSet(tagging.TagSet); err != nil {
h.logAndSendError(w, "some tags are invalid", reqInfo, err)
tagSet, err := readTagSet(r.Body)
if err != nil {
h.logAndSendError(w, "could not read tag set", reqInfo, err)
return
}
@ -44,11 +42,6 @@ func (h *handler) PutObjectTaggingHandler(w http.ResponseWriter, r *http.Request
return
}
tagSet := make(map[string]string, len(tagging.TagSet))
for _, tag := range tagging.TagSet {
tagSet[tag.Key] = tag.Value
}
p2 := &layer.PutTaggingParams{
ObjectInfo: objInfo,
TagSet: tagSet,
@ -81,31 +74,12 @@ func (h *handler) GetObjectTaggingHandler(w http.ResponseWriter, r *http.Request
return
}
tagging := &Tagging{}
for k, v := range tagSet {
tagging.TagSet = append(tagging.TagSet, Tag{Key: k, Value: v})
}
w.Header().Set(api.AmzVersionID, objInfo.Version())
if err = api.EncodeToResponse(w, tagging); err != nil {
if err = api.EncodeToResponse(w, encodeTagging(tagSet)); err != nil {
h.logAndSendError(w, "something went wrong", reqInfo, err)
}
}
func checkTagSet(tagSet []Tag) error {
if len(tagSet) > 10 {
return errors.GetAPIError(errors.ErrInvalidTag)
}
for _, tag := range tagSet {
if err := checkTag(tag); err != nil {
return err
}
}
return nil
}
func (h *handler) DeleteObjectTaggingHandler(w http.ResponseWriter, r *http.Request) {
reqInfo := api.GetReqInfo(r.Context())
@ -128,34 +102,114 @@ func (h *handler) DeleteObjectTaggingHandler(w http.ResponseWriter, r *http.Requ
w.WriteHeader(http.StatusNoContent)
}
func (h *handler) PutBucketTaggingHandler(w http.ResponseWriter, r *http.Request) {
reqInfo := api.GetReqInfo(r.Context())
tagSet, err := readTagSet(r.Body)
if err != nil {
h.logAndSendError(w, "could not read tag set", reqInfo, err)
return
}
if err := h.obj.PutBucketTagging(r.Context(), reqInfo.BucketName, tagSet); err != nil {
h.logAndSendError(w, "could not put object tagging", reqInfo, err)
}
}
func (h *handler) GetBucketTaggingHandler(w http.ResponseWriter, r *http.Request) {
reqInfo := api.GetReqInfo(r.Context())
tagSet, err := h.obj.GetBucketTagging(r.Context(), reqInfo.BucketName)
if err != nil {
h.logAndSendError(w, "could not get object tagging", reqInfo, err)
return
}
if err = api.EncodeToResponse(w, encodeTagging(tagSet)); err != nil {
h.logAndSendError(w, "something went wrong", reqInfo, err)
}
}
func (h *handler) DeleteBucketTaggingHandler(w http.ResponseWriter, r *http.Request) {
reqInfo := api.GetReqInfo(r.Context())
if err := h.obj.DeleteBucketTagging(r.Context(), reqInfo.BucketName); err != nil {
h.logAndSendError(w, "could not delete object tagging", reqInfo, err)
}
w.WriteHeader(http.StatusNoContent)
}
func readTagSet(reader io.Reader) (map[string]string, error) {
tagging := new(Tagging)
if err := xml.NewDecoder(reader).Decode(tagging); err != nil {
return nil, errors.GetAPIError(errors.ErrMalformedXML)
}
if err := checkTagSet(tagging.TagSet); err != nil {
return nil, err
}
tagSet := make(map[string]string, len(tagging.TagSet))
for _, tag := range tagging.TagSet {
tagSet[tag.Key] = tag.Value
}
return tagSet, nil
}
func encodeTagging(tagSet map[string]string) *Tagging {
tagging := &Tagging{}
for k, v := range tagSet {
tagging.TagSet = append(tagging.TagSet, Tag{Key: k, Value: v})
}
sort.Slice(tagging.TagSet, func(i, j int) bool {
return tagging.TagSet[i].Key < tagging.TagSet[j].Key
})
return tagging
}
func checkTagSet(tagSet []Tag) error {
if len(tagSet) > maxTags {
return errors.GetAPIError(errors.ErrInvalidTagsSizeExceed)
}
for _, tag := range tagSet {
if err := checkTag(tag); err != nil {
return err
}
}
return nil
}
func checkTag(tag Tag) error {
if len(tag.Key) < 1 || len(tag.Key) > keyTagMaxLength {
return errors.GetAPIError(errors.ErrInvalidTag)
return errors.GetAPIError(errors.ErrInvalidTagKey)
}
if len(tag.Value) < 1 || len(tag.Value) > valueTagMaxLength {
return errors.GetAPIError(errors.ErrInvalidTag)
if len(tag.Value) > valueTagMaxLength {
return errors.GetAPIError(errors.ErrInvalidTagValue)
}
if strings.HasPrefix(tag.Key, "aws:") {
return errors.GetAPIError(errors.ErrInvalidTag)
return errors.GetAPIError(errors.ErrInvalidTagKey)
}
if err := checkCharacters(tag.Key); err != nil {
return err
if !isValidTag(tag.Key) {
return errors.GetAPIError(errors.ErrInvalidTagKey)
}
if err := checkCharacters(tag.Value); err != nil {
return err
if !isValidTag(tag.Value) {
return errors.GetAPIError(errors.ErrInvalidTagValue)
}
return nil
}
func checkCharacters(str string) error {
func isValidTag(str string) bool {
for _, r := range str {
if !unicode.IsLetter(r) && !unicode.IsDigit(r) &&
!unicode.IsSpace(r) && !strings.ContainsRune(allowedTagChars, r) {
return errors.GetAPIError(errors.ErrInvalidTag)
return false
}
}
return nil
return true
}

View file

@ -23,7 +23,6 @@ func TestTagsValidity(t *testing.T) {
}{
{tag: Tag{}, valid: false},
{tag: Tag{Key: "", Value: "1"}, valid: false},
{tag: Tag{Key: "2", Value: ""}, valid: false},
{tag: Tag{Key: "aws:key", Value: "val"}, valid: false},
{tag: Tag{Key: "key~", Value: "val"}, valid: false},
{tag: Tag{Key: "key\\", Value: "val"}, valid: false},

View file

@ -83,18 +83,10 @@ func (h *handler) GetBucketReplicationHandler(w http.ResponseWriter, r *http.Req
h.logAndSendError(w, "not implemented", api.GetReqInfo(r.Context()), errors.GetAPIError(errors.ErrNotImplemented))
}
func (h *handler) GetBucketTaggingHandler(w http.ResponseWriter, r *http.Request) {
h.logAndSendError(w, "not implemented", api.GetReqInfo(r.Context()), errors.GetAPIError(errors.ErrNotImplemented))
}
func (h *handler) DeleteBucketWebsiteHandler(w http.ResponseWriter, r *http.Request) {
h.logAndSendError(w, "not implemented", api.GetReqInfo(r.Context()), errors.GetAPIError(errors.ErrNotImplemented))
}
func (h *handler) DeleteBucketTaggingHandler(w http.ResponseWriter, r *http.Request) {
h.logAndSendError(w, "not implemented", api.GetReqInfo(r.Context()), errors.GetAPIError(errors.ErrNotImplemented))
}
func (h *handler) GetBucketObjectLockConfigHandler(w http.ResponseWriter, r *http.Request) {
h.logAndSendError(w, "not implemented", api.GetReqInfo(r.Context()), errors.GetAPIError(errors.ErrNotImplemented))
}

View file

@ -6,6 +6,7 @@ const (
AmzMetadataDirective = "X-Amz-Metadata-Directive"
AmzVersionID = "X-Amz-Version-Id"
AmzTaggingCount = "X-Amz-Tagging-Count"
AmzTagging = "X-Amz-Tagging"
LastModified = "Last-Modified"
Date = "Date"

View file

@ -164,9 +164,11 @@ type (
GetObject(ctx context.Context, p *GetObjectParams) error
GetObjectInfo(ctx context.Context, p *HeadObjectParams) (*ObjectInfo, error)
GetObjectTagging(ctx context.Context, p *ObjectInfo) (map[string]string, error)
GetBucketTagging(ctx context.Context, bucket string) (map[string]string, error)
PutObject(ctx context.Context, p *PutObjectParams) (*ObjectInfo, error)
PutObjectTagging(ctx context.Context, p *PutTaggingParams) error
PutBucketTagging(ctx context.Context, bucket string, tagSet map[string]string) error
CopyObject(ctx context.Context, p *CopyObjectParams) (*ObjectInfo, error)
@ -176,10 +178,14 @@ type (
DeleteObjects(ctx context.Context, bucket string, objects []*VersionedObject) []error
DeleteObjectTagging(ctx context.Context, p *ObjectInfo) error
DeleteBucketTagging(ctx context.Context, bucket string) error
}
)
const tagPrefix = "S3-Tag-"
const (
tagPrefix = "S3-Tag-"
tagEmptyMark = "\\"
)
func (t *VersionedObject) String() string {
return t.Name + ":" + t.VersionID
@ -387,17 +393,38 @@ func (n *layer) GetObjectTagging(ctx context.Context, oi *ObjectInfo) (map[strin
return nil, err
}
return formTagSet(objInfo), nil
}
// GetBucketTagging from storage.
func (n *layer) GetBucketTagging(ctx context.Context, bucketName string) (map[string]string, error) {
bktInfo, err := n.GetBucketInfo(ctx, bucketName)
if err != nil {
return nil, err
}
objInfo, err := n.getSystemObject(ctx, bktInfo, formBucketTagObjectName(bucketName))
if err != nil && !errors.IsS3Error(err, errors.ErrNoSuchKey) {
return nil, err
}
return formTagSet(objInfo), nil
}
func formTagSet(objInfo *ObjectInfo) map[string]string {
var tagSet map[string]string
if objInfo != nil {
tagSet = make(map[string]string, len(objInfo.Headers))
for k, v := range objInfo.Headers {
if strings.HasPrefix(k, tagPrefix) {
if v == tagEmptyMark {
v = ""
}
tagSet[strings.TrimPrefix(k, tagPrefix)] = v
}
}
}
return tagSet, nil
return tagSet
}
// PutObjectTagging into storage.
@ -415,9 +442,27 @@ func (n *layer) PutObjectTagging(ctx context.Context, p *PutTaggingParams) error
return nil
}
// PutBucketTagging into storage.
func (n *layer) PutBucketTagging(ctx context.Context, bucketName string, tagSet map[string]string) error {
bktInfo, err := n.GetBucketInfo(ctx, bucketName)
if err != nil {
return err
}
if _, err = n.putSystemObject(ctx, bktInfo, formBucketTagObjectName(bucketName), tagSet, tagPrefix); err != nil {
return err
}
return nil
}
// DeleteObjectTagging from storage.
func (n *layer) DeleteObjectTagging(ctx context.Context, p *ObjectInfo) error {
oid, err := n.objectFindID(ctx, &findParams{cid: p.CID(), attr: objectSystemAttributeName, val: p.TagsObject()})
return n.deleteSystemObject(ctx, p.CID(), p.TagsObject())
}
func (n *layer) deleteSystemObject(ctx context.Context, bktCID *cid.ID, name string) error {
oid, err := n.objectFindID(ctx, &findParams{cid: bktCID, attr: objectSystemAttributeName, val: name})
if err != nil {
if errors.IsS3Error(err, errors.ErrNoSuchKey) {
return nil
@ -425,7 +470,17 @@ func (n *layer) DeleteObjectTagging(ctx context.Context, p *ObjectInfo) error {
return err
}
return n.objectDelete(ctx, p.CID(), oid)
return n.objectDelete(ctx, bktCID, oid)
}
// DeleteBucketTagging from storage.
func (n *layer) DeleteBucketTagging(ctx context.Context, bucketName string) error {
bktInfo, err := n.GetBucketInfo(ctx, bucketName)
if err != nil {
return err
}
return n.deleteSystemObject(ctx, bktInfo.CID, formBucketTagObjectName(bucketName))
}
func (n *layer) putSystemObject(ctx context.Context, bktInfo *cache.BucketInfo, objName string, metadata map[string]string, prefix string) (*object.ID, error) {
@ -453,6 +508,9 @@ func (n *layer) putSystemObject(ctx context.Context, bktInfo *cache.BucketInfo,
for k, v := range metadata {
attr := object.NewAttribute()
attr.SetKey(prefix + k)
if prefix == tagPrefix && v == "" {
v = tagEmptyMark
}
attr.SetValue(v)
attributes = append(attributes, attr)
}

View file

@ -198,3 +198,7 @@ func GetBoxData(ctx context.Context) (*accessbox.Box, error) {
}
return boxData, nil
}
func formBucketTagObjectName(name string) string {
return ".tagset." + name
}

View file

@ -209,9 +209,9 @@ See also `GetObject` and other method parameters.
| | Method | Comments |
|----|---------------------|----------|
| 🔴 | DeleteBucketTagging | |
| 🔴 | GetBucketTagging | |
| 🔴 | PutBucketTagging | |
| 🟢 | DeleteBucketTagging | |
| 🟢 | GetBucketTagging | |
| 🟢 | PutBucketTagging | |
## Tiering