TrueCloudLab/frostfs-s3-gw
17
3
Fork 19
Code Issues 30 Pull requests 7 Projects Releases 18 Packages Activity Actions

[#196] Add bucket tagging

Browse source 
Signed-off-by: Denis Kirillov <denis@nspcc.ru>
This commit is contained in:
Denis Kirillov 2021-08-17 14:57:24 +03:00 committed by Alex Vanin
parent 16da1aba64
commit 987185b9e1
10 changed files with 229 additions and 76 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

36
api/errors/errors.go
View file

@ -67,7 +67,10 @@ const (
ErrNoSuchUpload
ErrNoSuchVersion
ErrInvalidVersion
ErrInvalidTag
ErrInvalidArgument
ErrInvalidTagKey
ErrInvalidTagValue
ErrInvalidTagsSizeExceed
ErrNotImplemented
ErrPreconditionFailed
ErrNotModified
@ -300,7 +303,6 @@ const (
ErrEvaluatorInvalidTimestampFormatPatternSymbol
ErrEvaluatorBindingDoesNotExist
ErrMissingHeaders
ErrInvalidArgument
ErrInvalidColumnIndex
ErrAdminConfigNotificationTargetsFailed
@ -537,10 +539,28 @@ var errorCodes = errorCodeMap{
Description: "Invalid version id specified",
HTTPStatusCode: http.StatusBadRequest,
},
ErrInvalidTag: {
ErrCode: ErrInvalidTag,
ErrInvalidArgument: {
ErrCode: ErrInvalidArgument,
Code: "InvalidArgument",
Description: "The specified argument was invalid",
HTTPStatusCode: http.StatusBadRequest,
},
ErrInvalidTagKey: {
ErrCode: ErrInvalidTagKey,
Code: "InvalidTag",
Description: "You have passed bad tag input - duplicate keys, key/values are too long, system tags were sent.",
Description: "The TagValue you have provided is invalid",
HTTPStatusCode: http.StatusBadRequest,
},
ErrInvalidTagValue: {
ErrCode: ErrInvalidTagValue,
Code: "InvalidTag",
Description: "The TagKey you have provided is invalid",
HTTPStatusCode: http.StatusBadRequest,
},
ErrInvalidTagsSizeExceed: {
ErrCode: ErrInvalidTagsSizeExceed,
Code: "BadRequest",
Description: "Object tags cannot be greater than 10",
HTTPStatusCode: http.StatusBadRequest,
},
ErrNotImplemented: {
@ -1853,12 +1873,6 @@ var errorCodes = errorCodeMap{
Description: "Some headers in the query are missing from the file. Check the file and try again.",
HTTPStatusCode: http.StatusBadRequest,
},
ErrInvalidArgument: {
ErrCode: ErrInvalidArgument,
Code: "InvalidArgument",
Description: "The specified argument was invalid.",
HTTPStatusCode: http.StatusBadRequest,
},
ErrInvalidColumnIndex: {
ErrCode: ErrInvalidColumnIndex,
Code: "InvalidColumnIndex",

4
api/handler/not_support.go
View file

@ -19,10 +19,6 @@ func (h *handler) DeleteBucketEncryptionHandler(w http.ResponseWriter, r *http.R
h.logAndSendError(w, "not supported", api.GetReqInfo(r.Context()), errors.GetAPIError(errors.ErrNotSupported))
}
func (h *handler) PutBucketTaggingHandler(w http.ResponseWriter, r *http.Request) {
h.logAndSendError(w, "not supported", api.GetReqInfo(r.Context()), errors.GetAPIError(errors.ErrNotSupported))
}
func (h *handler) PutBucketNotificationHandler(w http.ResponseWriter, r *http.Request) {
h.logAndSendError(w, "not supported", api.GetReqInfo(r.Context()), errors.GetAPIError(errors.ErrNotSupported))
}

35
api/handler/put.go
View file

@ -4,6 +4,7 @@ import (
"encoding/xml"
"net"
"net/http"
"net/url"
"strings"
"github.com/nspcc-dev/neofs-api-go/pkg/acl/eacl"
@ -31,6 +32,11 @@ type createBucketParams struct {
func (h *handler) PutObjectHandler(w http.ResponseWriter, r *http.Request) {
var newEaclTable *eacl.Table
reqInfo := api.GetReqInfo(r.Context())
tagSet, err := parseTaggingHeader(r.Header)
if err != nil {
h.logAndSendError(w, "could not parse tagging header", reqInfo, err)
return
}
if containsACLHeaders(r) {
objectACL, err := parseACLHeaders(r)
@ -102,6 +108,13 @@ func (h *handler) PutObjectHandler(w http.ResponseWriter, r *http.Request) {
return
}
if tagSet != nil {
if err = h.obj.PutObjectTagging(r.Context(), &layer.PutTaggingParams{ObjectInfo: info, TagSet: tagSet}); err != nil {
h.logAndSendError(w, "could not upload object tagging", reqInfo, err)
return
}
}
if newEaclTable != nil {
p := &layer.PutBucketACLParams{
Name: reqInfo.BucketName,
@ -129,6 +142,28 @@ func containsACLHeaders(r *http.Request) bool {
r.Header.Get(api.AmzGrantFullControl) != "" || r.Header.Get(api.AmzGrantWrite) != ""
}
func parseTaggingHeader(header http.Header) (map[string]string, error) {
var tagSet map[string]string
if tagging := header.Get(api.AmzTagging); len(tagging) > 0 {
queries, err := url.ParseQuery(tagging)
if err != nil {
return nil, errors.GetAPIError(errors.ErrInvalidArgument)
}
if len(queries) > maxTags {
return nil, errors.GetAPIError(errors.ErrInvalidTagsSizeExceed)
}
tagSet = make(map[string]string, len(queries))
for k, v := range queries {
tag := Tag{Key: k, Value: v[0]}
if err = checkTag(tag); err != nil {
return nil, err
}
tagSet[tag.Key] = tag.Value
}
}
return tagSet, nil
}
func parseMetadata(r *http.Request) map[string]string {
res := make(map[string]string)
for k, v := range r.Header {

142
api/handler/tagging.go
View file

@ -2,7 +2,9 @@ package handler
import (
"encoding/xml"
"io"
"net/http"
"sort"
"strings"
"unicode"
@ -14,6 +16,7 @@ import (
const (
allowedTagChars = "+-=._:/@"
maxTags = 10
keyTagMaxLength = 128
valueTagMaxLength = 256
)
@ -21,14 +24,9 @@ const (
func (h *handler) PutObjectTaggingHandler(w http.ResponseWriter, r *http.Request) {
reqInfo := api.GetReqInfo(r.Context())
tagging := new(Tagging)
if err := xml.NewDecoder(r.Body).Decode(tagging); err != nil {
h.logAndSendError(w, "could not decode body", reqInfo, errors.GetAPIError(errors.ErrMalformedXML))
return
}
if err := checkTagSet(tagging.TagSet); err != nil {
h.logAndSendError(w, "some tags are invalid", reqInfo, err)
tagSet, err := readTagSet(r.Body)
if err != nil {
h.logAndSendError(w, "could not read tag set", reqInfo, err)
return
}
@ -44,11 +42,6 @@ func (h *handler) PutObjectTaggingHandler(w http.ResponseWriter, r *http.Request
return
}
tagSet := make(map[string]string, len(tagging.TagSet))
for _, tag := range tagging.TagSet {
tagSet[tag.Key] = tag.Value
}
p2 := &layer.PutTaggingParams{
ObjectInfo: objInfo,
TagSet: tagSet,
@ -81,31 +74,12 @@ func (h *handler) GetObjectTaggingHandler(w http.ResponseWriter, r *http.Request
return
}
tagging := &Tagging{}
for k, v := range tagSet {
tagging.TagSet = append(tagging.TagSet, Tag{Key: k, Value: v})
}
w.Header().Set(api.AmzVersionID, objInfo.Version())
if err = api.EncodeToResponse(w, tagging); err != nil {
if err = api.EncodeToResponse(w, encodeTagging(tagSet)); err != nil {
h.logAndSendError(w, "something went wrong", reqInfo, err)
}
}
func checkTagSet(tagSet []Tag) error {
if len(tagSet) > 10 {
return errors.GetAPIError(errors.ErrInvalidTag)
}
for _, tag := range tagSet {
if err := checkTag(tag); err != nil {
return err
}
}
return nil
}
func (h *handler) DeleteObjectTaggingHandler(w http.ResponseWriter, r *http.Request) {
reqInfo := api.GetReqInfo(r.Context())
@ -128,34 +102,114 @@ func (h *handler) DeleteObjectTaggingHandler(w http.ResponseWriter, r *http.Requ
w.WriteHeader(http.StatusNoContent)
}
func (h *handler) PutBucketTaggingHandler(w http.ResponseWriter, r *http.Request) {
reqInfo := api.GetReqInfo(r.Context())
tagSet, err := readTagSet(r.Body)
if err != nil {
h.logAndSendError(w, "could not read tag set", reqInfo, err)
return
}
if err := h.obj.PutBucketTagging(r.Context(), reqInfo.BucketName, tagSet); err != nil {
h.logAndSendError(w, "could not put object tagging", reqInfo, err)
}
}
func (h *handler) GetBucketTaggingHandler(w http.ResponseWriter, r *http.Request) {
reqInfo := api.GetReqInfo(r.Context())
tagSet, err := h.obj.GetBucketTagging(r.Context(), reqInfo.BucketName)
if err != nil {
h.logAndSendError(w, "could not get object tagging", reqInfo, err)
return
}
if err = api.EncodeToResponse(w, encodeTagging(tagSet)); err != nil {
h.logAndSendError(w, "something went wrong", reqInfo, err)
}
}
func (h *handler) DeleteBucketTaggingHandler(w http.ResponseWriter, r *http.Request) {
reqInfo := api.GetReqInfo(r.Context())
if err := h.obj.DeleteBucketTagging(r.Context(), reqInfo.BucketName); err != nil {
h.logAndSendError(w, "could not delete object tagging", reqInfo, err)
}
w.WriteHeader(http.StatusNoContent)
}
func readTagSet(reader io.Reader) (map[string]string, error) {
tagging := new(Tagging)
if err := xml.NewDecoder(reader).Decode(tagging); err != nil {
return nil, errors.GetAPIError(errors.ErrMalformedXML)
}
if err := checkTagSet(tagging.TagSet); err != nil {
return nil, err
}
tagSet := make(map[string]string, len(tagging.TagSet))
for _, tag := range tagging.TagSet {
tagSet[tag.Key] = tag.Value
}
return tagSet, nil
}
func encodeTagging(tagSet map[string]string) *Tagging {
tagging := &Tagging{}
for k, v := range tagSet {
tagging.TagSet = append(tagging.TagSet, Tag{Key: k, Value: v})
}
sort.Slice(tagging.TagSet, func(i, j int) bool {
return tagging.TagSet[i].Key < tagging.TagSet[j].Key
})
return tagging
}
func checkTagSet(tagSet []Tag) error {
if len(tagSet) > maxTags {
return errors.GetAPIError(errors.ErrInvalidTagsSizeExceed)
}
for _, tag := range tagSet {
if err := checkTag(tag); err != nil {
return err
}
}
return nil
}
func checkTag(tag Tag) error {
if len(tag.Key) < 1 || len(tag.Key) > keyTagMaxLength {
return errors.GetAPIError(errors.ErrInvalidTag)
return errors.GetAPIError(errors.ErrInvalidTagKey)
}
if len(tag.Value) < 1 || len(tag.Value) > valueTagMaxLength {
return errors.GetAPIError(errors.ErrInvalidTag)
if len(tag.Value) > valueTagMaxLength {
return errors.GetAPIError(errors.ErrInvalidTagValue)
}
if strings.HasPrefix(tag.Key, "aws:") {
return errors.GetAPIError(errors.ErrInvalidTag)
return errors.GetAPIError(errors.ErrInvalidTagKey)
}
if err := checkCharacters(tag.Key); err != nil {
return err
if !isValidTag(tag.Key) {
return errors.GetAPIError(errors.ErrInvalidTagKey)
}
if err := checkCharacters(tag.Value); err != nil {
return err
if !isValidTag(tag.Value) {
return errors.GetAPIError(errors.ErrInvalidTagValue)
}
return nil
}
func checkCharacters(str string) error {
func isValidTag(str string) bool {
for _, r := range str {
if !unicode.IsLetter(r) && !unicode.IsDigit(r) &&
!unicode.IsSpace(r) && !strings.ContainsRune(allowedTagChars, r) {
return errors.GetAPIError(errors.ErrInvalidTag)
return false
}
}
return nil
return true
}

1
api/handler/tagging_test.go
View file

@ -23,7 +23,6 @@ func TestTagsValidity(t *testing.T) {
}{
{tag: Tag{}, valid: false},
{tag: Tag{Key: "", Value: "1"}, valid: false},
{tag: Tag{Key: "2", Value: ""}, valid: false},
{tag: Tag{Key: "aws:key", Value: "val"}, valid: false},
{tag: Tag{Key: "key~", Value: "val"}, valid: false},
{tag: Tag{Key: "key\\", Value: "val"}, valid: false},

8
api/handler/unimplemented.go
View file

@ -83,18 +83,10 @@ func (h *handler) GetBucketReplicationHandler(w http.ResponseWriter, r *http.Req
h.logAndSendError(w, "not implemented", api.GetReqInfo(r.Context()), errors.GetAPIError(errors.ErrNotImplemented))
}
func (h *handler) GetBucketTaggingHandler(w http.ResponseWriter, r *http.Request) {
h.logAndSendError(w, "not implemented", api.GetReqInfo(r.Context()), errors.GetAPIError(errors.ErrNotImplemented))
}
func (h *handler) DeleteBucketWebsiteHandler(w http.ResponseWriter, r *http.Request) {
h.logAndSendError(w, "not implemented", api.GetReqInfo(r.Context()), errors.GetAPIError(errors.ErrNotImplemented))
}
func (h *handler) DeleteBucketTaggingHandler(w http.ResponseWriter, r *http.Request) {
h.logAndSendError(w, "not implemented", api.GetReqInfo(r.Context()), errors.GetAPIError(errors.ErrNotImplemented))
}
func (h *handler) GetBucketObjectLockConfigHandler(w http.ResponseWriter, r *http.Request) {
h.logAndSendError(w, "not implemented", api.GetReqInfo(r.Context()), errors.GetAPIError(errors.ErrNotImplemented))
}

1
api/headers.go
View file

@ -6,6 +6,7 @@ const (
AmzMetadataDirective = "X-Amz-Metadata-Directive"
AmzVersionID = "X-Amz-Version-Id"
AmzTaggingCount = "X-Amz-Tagging-Count"
AmzTagging = "X-Amz-Tagging"
LastModified = "Last-Modified"
Date = "Date"

68
api/layer/layer.go
View file

@ -164,9 +164,11 @@ type (
GetObject(ctx context.Context, p *GetObjectParams) error
GetObjectInfo(ctx context.Context, p *HeadObjectParams) (*ObjectInfo, error)
GetObjectTagging(ctx context.Context, p *ObjectInfo) (map[string]string, error)
GetBucketTagging(ctx context.Context, bucket string) (map[string]string, error)
PutObject(ctx context.Context, p *PutObjectParams) (*ObjectInfo, error)
PutObjectTagging(ctx context.Context, p *PutTaggingParams) error
PutBucketTagging(ctx context.Context, bucket string, tagSet map[string]string) error
CopyObject(ctx context.Context, p *CopyObjectParams) (*ObjectInfo, error)
@ -176,10 +178,14 @@ type (
DeleteObjects(ctx context.Context, bucket string, objects []*VersionedObject) []error
DeleteObjectTagging(ctx context.Context, p *ObjectInfo) error
DeleteBucketTagging(ctx context.Context, bucket string) error
}
)
const tagPrefix = "S3-Tag-"
const (
tagPrefix = "S3-Tag-"
tagEmptyMark = "\\"
)
func (t *VersionedObject) String() string {
return t.Name + ":" + t.VersionID
@ -387,17 +393,38 @@ func (n *layer) GetObjectTagging(ctx context.Context, oi *ObjectInfo) (map[strin
return nil, err
}
return formTagSet(objInfo), nil
}
// GetBucketTagging from storage.
func (n *layer) GetBucketTagging(ctx context.Context, bucketName string) (map[string]string, error) {
bktInfo, err := n.GetBucketInfo(ctx, bucketName)
if err != nil {
return nil, err
}
objInfo, err := n.getSystemObject(ctx, bktInfo, formBucketTagObjectName(bucketName))
if err != nil && !errors.IsS3Error(err, errors.ErrNoSuchKey) {
return nil, err
}
return formTagSet(objInfo), nil
}
func formTagSet(objInfo *ObjectInfo) map[string]string {
var tagSet map[string]string
if objInfo != nil {
tagSet = make(map[string]string, len(objInfo.Headers))
for k, v := range objInfo.Headers {
if strings.HasPrefix(k, tagPrefix) {
if v == tagEmptyMark {
v = ""
}
tagSet[strings.TrimPrefix(k, tagPrefix)] = v
}
}
}
return tagSet, nil
return tagSet
}
// PutObjectTagging into storage.
@ -415,9 +442,27 @@ func (n *layer) PutObjectTagging(ctx context.Context, p *PutTaggingParams) error
return nil
}
// PutBucketTagging into storage.
func (n *layer) PutBucketTagging(ctx context.Context, bucketName string, tagSet map[string]string) error {
bktInfo, err := n.GetBucketInfo(ctx, bucketName)
if err != nil {
return err
}
if _, err = n.putSystemObject(ctx, bktInfo, formBucketTagObjectName(bucketName), tagSet, tagPrefix); err != nil {
return err
}
return nil
}
// DeleteObjectTagging from storage.
func (n *layer) DeleteObjectTagging(ctx context.Context, p *ObjectInfo) error {
oid, err := n.objectFindID(ctx, &findParams{cid: p.CID(), attr: objectSystemAttributeName, val: p.TagsObject()})
return n.deleteSystemObject(ctx, p.CID(), p.TagsObject())
}
func (n *layer) deleteSystemObject(ctx context.Context, bktCID *cid.ID, name string) error {
oid, err := n.objectFindID(ctx, &findParams{cid: bktCID, attr: objectSystemAttributeName, val: name})
if err != nil {
if errors.IsS3Error(err, errors.ErrNoSuchKey) {
return nil
@ -425,7 +470,17 @@ func (n *layer) DeleteObjectTagging(ctx context.Context, p *ObjectInfo) error {
return err
}
return n.objectDelete(ctx, p.CID(), oid)
return n.objectDelete(ctx, bktCID, oid)
}
// DeleteBucketTagging from storage.
func (n *layer) DeleteBucketTagging(ctx context.Context, bucketName string) error {
bktInfo, err := n.GetBucketInfo(ctx, bucketName)
if err != nil {
return err
}
return n.deleteSystemObject(ctx, bktInfo.CID, formBucketTagObjectName(bucketName))
}
func (n *layer) putSystemObject(ctx context.Context, bktInfo *cache.BucketInfo, objName string, metadata map[string]string, prefix string) (*object.ID, error) {
@ -453,6 +508,9 @@ func (n *layer) putSystemObject(ctx context.Context, bktInfo *cache.BucketInfo,
for k, v := range metadata {
attr := object.NewAttribute()
attr.SetKey(prefix + k)
if prefix == tagPrefix && v == "" {
v = tagEmptyMark
}
attr.SetValue(v)
attributes = append(attributes, attr)
}

4
api/layer/util.go
View file

@ -198,3 +198,7 @@ func GetBoxData(ctx context.Context) (*accessbox.Box, error) {
}
return boxData, nil
}
func formBucketTagObjectName(name string) string {
return ".tagset." + name
}

6
docs/aws_s3_compat.md
View file

@ -209,9 +209,9 @@ See also `GetObject` and other method parameters.
| | Method | Comments |
|----|---------------------|----------|
| 🔴 | DeleteBucketTagging | |
| 🔴 | GetBucketTagging | |
| 🔴 | PutBucketTagging | |
| 🟢 | DeleteBucketTagging | |
| 🟢 | GetBucketTagging | |
| 🟢 | PutBucketTagging | |
## Tiering