From 8b5ebe2ec21964f26d09974806fca7888d0511f3 Mon Sep 17 00:00:00 2001 From: Denis Kirillov Date: Tue, 17 Aug 2021 11:04:42 +0300 Subject: [PATCH] [#196] Add PutObjectTagging Signed-off-by: Denis Kirillov --- api/errors/errors.go | 7 +++ api/handler/response.go | 12 ++++ api/handler/tagging.go | 107 +++++++++++++++++++++++++++++++++++ api/handler/tagging_test.go | 47 +++++++++++++++ api/handler/unimplemented.go | 4 -- api/layer/layer.go | 77 +++++++++++++++++++++++++ api/layer/object.go | 9 +++ api/layer/util.go | 3 + docs/aws_s3_compat.md | 2 +- 9 files changed, 263 insertions(+), 5 deletions(-) create mode 100644 api/handler/tagging.go create mode 100644 api/handler/tagging_test.go diff --git a/api/errors/errors.go b/api/errors/errors.go index f9c8aadb..104b0d33 100644 --- a/api/errors/errors.go +++ b/api/errors/errors.go @@ -67,6 +67,7 @@ const ( ErrNoSuchUpload ErrNoSuchVersion ErrInvalidVersion + ErrInvalidTag ErrNotImplemented ErrPreconditionFailed ErrNotModified @@ -536,6 +537,12 @@ var errorCodes = errorCodeMap{ Description: "Invalid version id specified", HTTPStatusCode: http.StatusBadRequest, }, + ErrInvalidTag: { + ErrCode: ErrInvalidTag, + Code: "InvalidTag", + Description: "You have passed bad tag input - duplicate keys, key/values are too long, system tags were sent.", + HTTPStatusCode: http.StatusBadRequest, + }, ErrNotImplemented: { ErrCode: ErrNotImplemented, Code: "NotImplemented", diff --git a/api/handler/response.go b/api/handler/response.go index a8884063..918b37fa 100644 --- a/api/handler/response.go +++ b/api/handler/response.go @@ -171,6 +171,18 @@ type VersioningConfiguration struct { MfaDelete string `xml:"MfaDelete,omitempty"` } +// Tagging contains tag set. +type Tagging struct { + XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Tagging"` + TagSet []Tag `xml:"TagSet>Tag"` +} + +// Tag is AWS key-value tag. +type Tag struct { + Key string + Value string +} + // MarshalXML - StringMap marshals into XML. func (s StringMap) MarshalXML(e *xml.Encoder, start xml.StartElement) error { tokens := []xml.Token{start} diff --git a/api/handler/tagging.go b/api/handler/tagging.go new file mode 100644 index 00000000..19bb4291 --- /dev/null +++ b/api/handler/tagging.go @@ -0,0 +1,107 @@ +package handler + +import ( + "encoding/xml" + "net/http" + "strings" + "unicode" + + "github.com/nspcc-dev/neofs-s3-gw/api" + "github.com/nspcc-dev/neofs-s3-gw/api/errors" + "github.com/nspcc-dev/neofs-s3-gw/api/layer" +) + +const ( + allowedTagChars = "+-=._:/@" + + keyTagMaxLength = 128 + valueTagMaxLength = 256 +) + +func (h *handler) PutObjectTaggingHandler(w http.ResponseWriter, r *http.Request) { + reqInfo := api.GetReqInfo(r.Context()) + + tagging := new(Tagging) + if err := xml.NewDecoder(r.Body).Decode(tagging); err != nil { + h.logAndSendError(w, "could not decode body", reqInfo, errors.GetAPIError(errors.ErrMalformedXML)) + return + } + + if err := checkTagSet(tagging.TagSet); err != nil { + h.logAndSendError(w, "some tags are invalid", reqInfo, err) + return + } + + p := &layer.HeadObjectParams{ + Bucket: reqInfo.BucketName, + Object: reqInfo.ObjectName, + VersionID: reqInfo.URL.Query().Get("versionId"), + } + + objInfo, err := h.obj.GetObjectInfo(r.Context(), p) + if err != nil { + h.logAndSendError(w, "could not get object info", reqInfo, err) + return + } + + tagSet := make(map[string]string, len(tagging.TagSet)) + for _, tag := range tagging.TagSet { + tagSet[tag.Key] = tag.Value + } + + p2 := &layer.PutTaggingParams{ + ObjectInfo: objInfo, + TagSet: tagSet, + } + + if err = h.obj.PutObjectTagging(r.Context(), p2); err != nil { + h.logAndSendError(w, "could not put object tagging", reqInfo, err) + return + } +} + +func checkTagSet(tagSet []Tag) error { + if len(tagSet) > 10 { + return errors.GetAPIError(errors.ErrInvalidTag) + } + + for _, tag := range tagSet { + if err := checkTag(tag); err != nil { + return err + } + } + + return nil +} + +func checkTag(tag Tag) error { + if len(tag.Key) < 1 || len(tag.Key) > keyTagMaxLength { + return errors.GetAPIError(errors.ErrInvalidTag) + } + if len(tag.Value) < 1 || len(tag.Value) > valueTagMaxLength { + return errors.GetAPIError(errors.ErrInvalidTag) + } + + if strings.HasPrefix(tag.Key, "aws:") { + return errors.GetAPIError(errors.ErrInvalidTag) + } + + if err := checkCharacters(tag.Key); err != nil { + return err + } + if err := checkCharacters(tag.Value); err != nil { + return err + } + + return nil +} + +func checkCharacters(str string) error { + for _, r := range str { + if !unicode.IsLetter(r) && !unicode.IsDigit(r) && + !unicode.IsSpace(r) && !strings.ContainsRune(allowedTagChars, r) { + return errors.GetAPIError(errors.ErrInvalidTag) + } + } + return nil +} diff --git a/api/handler/tagging_test.go b/api/handler/tagging_test.go new file mode 100644 index 00000000..e305df95 --- /dev/null +++ b/api/handler/tagging_test.go @@ -0,0 +1,47 @@ +package handler + +import ( + "strings" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestTagsValidity(t *testing.T) { + sbKey := strings.Builder{} + for i := 0; i < keyTagMaxLength; i++ { + sbKey.WriteByte('a') + } + sbValue := strings.Builder{} + for i := 0; i < valueTagMaxLength; i++ { + sbValue.WriteByte('a') + } + + for _, tc := range []struct { + tag Tag + valid bool + }{ + {tag: Tag{}, valid: false}, + {tag: Tag{Key: "", Value: "1"}, valid: false}, + {tag: Tag{Key: "2", Value: ""}, valid: false}, + {tag: Tag{Key: "aws:key", Value: "val"}, valid: false}, + {tag: Tag{Key: "key~", Value: "val"}, valid: false}, + {tag: Tag{Key: "key\\", Value: "val"}, valid: false}, + {tag: Tag{Key: "key?", Value: "val"}, valid: false}, + {tag: Tag{Key: sbKey.String() + "b", Value: "val"}, valid: false}, + {tag: Tag{Key: "key", Value: sbValue.String() + "b"}, valid: false}, + + {tag: Tag{Key: sbKey.String(), Value: "val"}, valid: true}, + {tag: Tag{Key: "key", Value: sbValue.String()}, valid: true}, + {tag: Tag{Key: "k e y", Value: "v a l"}, valid: true}, + {tag: Tag{Key: "12345", Value: "1234"}, valid: true}, + {tag: Tag{Key: allowedTagChars, Value: allowedTagChars}, valid: true}, + } { + err := checkTag(tc.tag) + if tc.valid { + require.NoError(t, err) + } else { + require.Error(t, err) + } + } +} diff --git a/api/handler/unimplemented.go b/api/handler/unimplemented.go index d1b0255d..7d531ea6 100644 --- a/api/handler/unimplemented.go +++ b/api/handler/unimplemented.go @@ -35,10 +35,6 @@ func (h *handler) GetObjectTaggingHandler(w http.ResponseWriter, r *http.Request h.logAndSendError(w, "not implemented", api.GetReqInfo(r.Context()), errors.GetAPIError(errors.ErrNotImplemented)) } -func (h *handler) PutObjectTaggingHandler(w http.ResponseWriter, r *http.Request) { - h.logAndSendError(w, "not implemented", api.GetReqInfo(r.Context()), errors.GetAPIError(errors.ErrNotImplemented)) -} - func (h *handler) DeleteObjectTaggingHandler(w http.ResponseWriter, r *http.Request) { h.logAndSendError(w, "not implemented", api.GetReqInfo(r.Context()), errors.GetAPIError(errors.ErrNotImplemented)) } diff --git a/api/layer/layer.go b/api/layer/layer.go index 1f9614ad..763c0d00 100644 --- a/api/layer/layer.go +++ b/api/layer/layer.go @@ -7,6 +7,7 @@ import ( "fmt" "io" "net/url" + "strconv" "time" "github.com/nspcc-dev/neofs-api-go/pkg/acl/eacl" @@ -134,6 +135,12 @@ type ( VersionID string } + // PutTaggingParams stores tag set params. + PutTaggingParams struct { + ObjectInfo *ObjectInfo + TagSet map[string]string + } + // NeoFS provides basic NeoFS interface. NeoFS interface { Get(ctx context.Context, address *object.Address) (*object.Object, error) @@ -157,6 +164,7 @@ type ( GetObjectInfo(ctx context.Context, p *HeadObjectParams) (*ObjectInfo, error) PutObject(ctx context.Context, p *PutObjectParams) (*ObjectInfo, error) + PutObjectTagging(ctx context.Context, p *PutTaggingParams) error CopyObject(ctx context.Context, p *CopyObjectParams) (*ObjectInfo, error) @@ -168,6 +176,8 @@ type ( } ) +const tagPrefix = "S3-Tag-" + func (t *VersionedObject) String() string { return t.Name + ":" + t.VersionID } @@ -361,6 +371,73 @@ func (n *layer) PutObject(ctx context.Context, p *PutObjectParams) (*ObjectInfo, return n.objectPut(ctx, bkt, p) } +// PutObjectTagging into storage. +func (n *layer) PutObjectTagging(ctx context.Context, p *PutTaggingParams) error { + bktInfo := &cache.BucketInfo{ + Name: p.ObjectInfo.Bucket, + CID: p.ObjectInfo.CID(), + Owner: p.ObjectInfo.Owner, + } + + if _, err := n.putSystemObject(ctx, bktInfo, p.ObjectInfo.TagsObject(), p.TagSet, tagPrefix); err != nil { + return err + } + + return nil +} + +func (n *layer) putSystemObject(ctx context.Context, bktInfo *cache.BucketInfo, objName string, metadata map[string]string, prefix string) (*object.ID, error) { + oldOID, err := n.objectFindID(ctx, &findParams{cid: bktInfo.CID, attr: objectSystemAttributeName, val: objName}) + if err != nil && !errors.IsS3Error(err, errors.ErrNoSuchKey) { + return nil, err + } + + attributes := make([]*object.Attribute, 0, 3) + + filename := object.NewAttribute() + filename.SetKey(objectSystemAttributeName) + filename.SetValue(objName) + + createdAt := object.NewAttribute() + createdAt.SetKey(object.AttributeTimestamp) + createdAt.SetValue(strconv.FormatInt(time.Now().UTC().Unix(), 10)) + + versioningIgnore := object.NewAttribute() + versioningIgnore.SetKey(attrVersionsIgnore) + versioningIgnore.SetValue(strconv.FormatBool(true)) + + attributes = append(attributes, filename, createdAt, versioningIgnore) + + for k, v := range metadata { + attr := object.NewAttribute() + attr.SetKey(prefix + k) + attr.SetValue(v) + attributes = append(attributes, attr) + } + + raw := object.NewRaw() + raw.SetOwnerID(bktInfo.Owner) + raw.SetContainerID(bktInfo.CID) + raw.SetAttributes(attributes...) + + ops := new(client.PutObjectParams).WithObject(raw.Object()) + oid, err := n.pool.PutObject(ctx, ops, n.BearerOpt(ctx)) + if err != nil { + return nil, err + } + + if _, err = n.objectHead(ctx, bktInfo.CID, oid); err != nil { + return nil, err + } + if oldOID != nil { + if err = n.objectDelete(ctx, bktInfo.CID, oldOID); err != nil { + return nil, err + } + } + + return oid, nil +} + // CopyObject from one bucket into another bucket. func (n *layer) CopyObject(ctx context.Context, p *CopyObjectParams) (*ObjectInfo, error) { pr, pw := io.Pipe() diff --git a/api/layer/object.go b/api/layer/object.go index 92016bc2..ea2b3008 100644 --- a/api/layer/object.go +++ b/api/layer/object.go @@ -179,6 +179,15 @@ func (n *layer) objectPut(ctx context.Context, bkt *cache.BucketInfo, p *PutObje zap.Stringer("version id", id), zap.Error(err)) } + if !versioningEnabled { + if objVersion := versions.getVersion(id); objVersion != nil { + if err = n.DeleteObjectTagging(ctx, objVersion); err != nil { + n.log.Warn("couldn't delete object tagging", + zap.Stringer("version id", id), + zap.Error(err)) + } + } + } } return &ObjectInfo{ diff --git a/api/layer/util.go b/api/layer/util.go index 557141f7..7b5ad5d8 100644 --- a/api/layer/util.go +++ b/api/layer/util.go @@ -175,6 +175,9 @@ func (o *ObjectInfo) NiceName() string { return o.Bucket + "/" + o.Name } // Address returns object address. func (o *ObjectInfo) Address() *object.Address { return newAddress(o.bucketID, o.id) } +// TagsObject returns name of system object for tags. +func (o *ObjectInfo) TagsObject() string { return ".tagset." + o.Name + "." + o.Version() } + // CID returns bucket ID from ObjectInfo. func (o *ObjectInfo) CID() *cid.ID { return o.bucketID } diff --git a/docs/aws_s3_compat.md b/docs/aws_s3_compat.md index 647f8ce4..24e8aa68 100644 --- a/docs/aws_s3_compat.md +++ b/docs/aws_s3_compat.md @@ -70,7 +70,7 @@ Should be supported soon. |----|---------------------|----------| | 🔴 | DeleteObjectTagging | | | 🔴 | GetObjectTagging | | -| 🔴 | PutObjectTagging | | +| 🟢 | PutObjectTagging | | ## Versioning