diff --git a/api/data/info.go b/api/data/info.go index faa746e..041e001 100644 --- a/api/data/info.go +++ b/api/data/info.go @@ -83,6 +83,14 @@ type ( ExposeHeaders []string `xml:"ExposeHeader" json:"ExposeHeaders"` MaxAgeSeconds int `xml:"MaxAgeSeconds,omitempty" json:"MaxAgeSeconds,omitempty"` } + + // ObjectVersion stores object version info. + ObjectVersion struct { + BktInfo *BucketInfo + ObjectName string + VersionID string + NoErrorOnDeleteMarker bool + } ) // NotificationInfoFromObject creates new NotificationInfo from ObjectInfo. diff --git a/api/data/tagging.go b/api/data/tagging.go new file mode 100644 index 0000000..5a0472a --- /dev/null +++ b/api/data/tagging.go @@ -0,0 +1,30 @@ +package data + +import "encoding/xml" + +// Tagging contains tag set. +type Tagging struct { + XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Tagging"` + TagSet []Tag `xml:"TagSet>Tag"` +} + +// Tag is an AWS key-value tag. +type Tag struct { + Key string + Value string +} + +type GetObjectTaggingParams struct { + ObjectVersion *ObjectVersion + + // NodeVersion can be nil. If not nil we save one request to tree service. + NodeVersion *NodeVersion // optional +} + +type PutObjectTaggingParams struct { + ObjectVersion *ObjectVersion + TagSet map[string]string + + // NodeVersion can be nil. If not nil we save one request to tree service. + NodeVersion *NodeVersion // optional +} diff --git a/api/handler/copy.go b/api/handler/copy.go index ef35ef8..5b1fea8 100644 --- a/api/handler/copy.go +++ b/api/handler/copy.go @@ -168,8 +168,8 @@ func (h *handler) CopyObjectHandler(w http.ResponseWriter, r *http.Request) { return } } else { - tagPrm := &layer.GetObjectTaggingParams{ - ObjectVersion: &layer.ObjectVersion{ + tagPrm := &data.GetObjectTaggingParams{ + ObjectVersion: &data.ObjectVersion{ BktInfo: srcObjPrm.BktInfo, ObjectName: srcObject, VersionID: srcObjInfo.VersionID(), @@ -259,8 +259,8 @@ func (h *handler) CopyObjectHandler(w http.ResponseWriter, r *http.Request) { } if tagSet != nil { - tagPrm := &layer.PutObjectTaggingParams{ - ObjectVersion: &layer.ObjectVersion{ + tagPrm := &data.PutObjectTaggingParams{ + ObjectVersion: &data.ObjectVersion{ BktInfo: dstBktInfo, ObjectName: reqInfo.ObjectName, VersionID: dstObjInfo.VersionID(), diff --git a/api/handler/copy_test.go b/api/handler/copy_test.go index d6bdff2..6c25b79 100644 --- a/api/handler/copy_test.go +++ b/api/handler/copy_test.go @@ -11,9 +11,11 @@ import ( "testing" "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api" + "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data" "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors" "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer" "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer/encryption" + "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/middleware" "github.com/stretchr/testify/require" ) @@ -289,23 +291,24 @@ func copyObject(hc *handlerContext, bktName, fromObject, toObject string, copyMe } func putObjectTagging(t *testing.T, tc *handlerContext, bktName, objName string, tags map[string]string) { - body := &Tagging{ - TagSet: make([]Tag, 0, len(tags)), + body := &data.Tagging{ + TagSet: make([]data.Tag, 0, len(tags)), } for key, val := range tags { - body.TagSet = append(body.TagSet, Tag{ + body.TagSet = append(body.TagSet, data.Tag{ Key: key, Value: val, }) } w, r := prepareTestRequest(tc, bktName, objName, body) + middleware.GetReqInfo(r.Context()).Tagging = body tc.Handler().PutObjectTaggingHandler(w, r) assertStatus(t, w, http.StatusOK) } -func getObjectTagging(t *testing.T, tc *handlerContext, bktName, objName, version string) *Tagging { +func getObjectTagging(t *testing.T, tc *handlerContext, bktName, objName, version string) *data.Tagging { query := make(url.Values) query.Add(api.QueryVersionID, version) @@ -313,7 +316,7 @@ func getObjectTagging(t *testing.T, tc *handlerContext, bktName, objName, versio tc.Handler().GetObjectTaggingHandler(w, r) assertStatus(t, w, http.StatusOK) - tagging := &Tagging{} + tagging := &data.Tagging{} err := xml.NewDecoder(w.Result().Body).Decode(tagging) require.NoError(t, err) return tagging diff --git a/api/handler/get.go b/api/handler/get.go index e5419e4..acdc375 100644 --- a/api/handler/get.go +++ b/api/handler/get.go @@ -184,7 +184,7 @@ func (h *handler) GetObjectHandler(w http.ResponseWriter, r *http.Request) { return } - t := &layer.ObjectVersion{ + t := &data.ObjectVersion{ BktInfo: bktInfo, ObjectName: info.Name, VersionID: info.VersionID(), diff --git a/api/handler/head.go b/api/handler/head.go index 8a3bf66..15a408d 100644 --- a/api/handler/head.go +++ b/api/handler/head.go @@ -70,7 +70,7 @@ func (h *handler) HeadObjectHandler(w http.ResponseWriter, r *http.Request) { return } - t := &layer.ObjectVersion{ + t := &data.ObjectVersion{ BktInfo: bktInfo, ObjectName: info.Name, VersionID: info.VersionID(), diff --git a/api/handler/locking.go b/api/handler/locking.go index b9d6c14..fa07774 100644 --- a/api/handler/locking.go +++ b/api/handler/locking.go @@ -133,7 +133,7 @@ func (h *handler) PutObjectLegalHoldHandler(w http.ResponseWriter, r *http.Reque } p := &layer.PutLockInfoParams{ - ObjVersion: &layer.ObjectVersion{ + ObjVersion: &data.ObjectVersion{ BktInfo: bktInfo, ObjectName: reqInfo.ObjectName, VersionID: reqInfo.URL.Query().Get(api.QueryVersionID), @@ -172,7 +172,7 @@ func (h *handler) GetObjectLegalHoldHandler(w http.ResponseWriter, r *http.Reque return } - p := &layer.ObjectVersion{ + p := &data.ObjectVersion{ BktInfo: bktInfo, ObjectName: reqInfo.ObjectName, VersionID: reqInfo.URL.Query().Get(api.QueryVersionID), @@ -221,7 +221,7 @@ func (h *handler) PutObjectRetentionHandler(w http.ResponseWriter, r *http.Reque } p := &layer.PutLockInfoParams{ - ObjVersion: &layer.ObjectVersion{ + ObjVersion: &data.ObjectVersion{ BktInfo: bktInfo, ObjectName: reqInfo.ObjectName, VersionID: reqInfo.URL.Query().Get(api.QueryVersionID), @@ -256,7 +256,7 @@ func (h *handler) GetObjectRetentionHandler(w http.ResponseWriter, r *http.Reque return } - p := &layer.ObjectVersion{ + p := &data.ObjectVersion{ BktInfo: bktInfo, ObjectName: reqInfo.ObjectName, VersionID: reqInfo.URL.Query().Get(api.QueryVersionID), diff --git a/api/handler/multipart_upload.go b/api/handler/multipart_upload.go index fd1cc8a..72f7689 100644 --- a/api/handler/multipart_upload.go +++ b/api/handler/multipart_upload.go @@ -487,8 +487,8 @@ func (h *handler) completeMultipartUpload(r *http.Request, c *layer.CompleteMult objInfo := extendedObjInfo.ObjectInfo if len(uploadData.TagSet) != 0 { - tagPrm := &layer.PutObjectTaggingParams{ - ObjectVersion: &layer.ObjectVersion{ + tagPrm := &data.PutObjectTaggingParams{ + ObjectVersion: &data.ObjectVersion{ BktInfo: bktInfo, ObjectName: objInfo.Name, VersionID: objInfo.VersionID(), diff --git a/api/handler/put.go b/api/handler/put.go index 198f6b9..28d6635 100644 --- a/api/handler/put.go +++ b/api/handler/put.go @@ -307,8 +307,8 @@ func (h *handler) PutObjectHandler(w http.ResponseWriter, r *http.Request) { } if tagSet != nil { - tagPrm := &layer.PutObjectTaggingParams{ - ObjectVersion: &layer.ObjectVersion{ + tagPrm := &data.PutObjectTaggingParams{ + ObjectVersion: &data.ObjectVersion{ BktInfo: bktInfo, ObjectName: objInfo.Name, VersionID: objInfo.VersionID(), @@ -483,7 +483,12 @@ func (h *handler) PostObject(w http.ResponseWriter, r *http.Request) { if tagging := auth.MultipartFormValue(r, "tagging"); tagging != "" { buffer := bytes.NewBufferString(tagging) - tagSet, err = h.readTagSet(buffer) + tags := new(data.Tagging) + if err = h.cfg.NewXMLDecoder(buffer).Decode(tags); err != nil { + h.logAndSendError(w, "could not decode tag set", reqInfo, errors.GetAPIError(errors.ErrMalformedXML)) + return + } + tagSet, err = h.readTagSet(tags) if err != nil { h.logAndSendError(w, "could not read tag set", reqInfo, err) return @@ -574,8 +579,8 @@ func (h *handler) PostObject(w http.ResponseWriter, r *http.Request) { } if tagSet != nil { - tagPrm := &layer.PutObjectTaggingParams{ - ObjectVersion: &layer.ObjectVersion{ + tagPrm := &data.PutObjectTaggingParams{ + ObjectVersion: &data.ObjectVersion{ BktInfo: bktInfo, ObjectName: objInfo.Name, VersionID: objInfo.VersionID(), @@ -789,7 +794,7 @@ func parseTaggingHeader(header http.Header) (map[string]string, error) { } tagSet = make(map[string]string, len(queries)) for k, v := range queries { - tag := Tag{Key: k, Value: v[0]} + tag := data.Tag{Key: k, Value: v[0]} if err = checkTag(tag); err != nil { return nil, err } diff --git a/api/handler/response.go b/api/handler/response.go index e1ed08d..9ff2455 100644 --- a/api/handler/response.go +++ b/api/handler/response.go @@ -185,12 +185,6 @@ type VersioningConfiguration struct { MfaDelete string `xml:"MfaDelete,omitempty"` } -// Tagging contains tag set. -type Tagging struct { - XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Tagging"` - TagSet []Tag `xml:"TagSet>Tag"` -} - // PostResponse contains result of posting object. type PostResponse struct { Bucket string `xml:"Bucket"` @@ -198,12 +192,6 @@ type PostResponse struct { ETag string `xml:"Etag"` } -// Tag is an AWS key-value tag. -type Tag struct { - Key string - Value string -} - // MarshalXML -- StringMap marshals into XML. func (s StringMap) MarshalXML(e *xml.Encoder, start xml.StartElement) error { tokens := []xml.Token{start} diff --git a/api/handler/tagging.go b/api/handler/tagging.go index 1025803..3900e75 100644 --- a/api/handler/tagging.go +++ b/api/handler/tagging.go @@ -1,7 +1,6 @@ package handler import ( - "io" "net/http" "sort" "strings" @@ -10,7 +9,6 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api" "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data" "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors" - "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer" "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/middleware" "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs" "go.uber.org/zap" @@ -28,7 +26,7 @@ func (h *handler) PutObjectTaggingHandler(w http.ResponseWriter, r *http.Request ctx := r.Context() reqInfo := middleware.GetReqInfo(ctx) - tagSet, err := h.readTagSet(r.Body) + tagSet, err := h.readTagSet(reqInfo.Tagging) if err != nil { h.logAndSendError(w, "could not read tag set", reqInfo, err) return @@ -40,8 +38,8 @@ func (h *handler) PutObjectTaggingHandler(w http.ResponseWriter, r *http.Request return } - tagPrm := &layer.PutObjectTaggingParams{ - ObjectVersion: &layer.ObjectVersion{ + tagPrm := &data.PutObjectTaggingParams{ + ObjectVersion: &data.ObjectVersion{ BktInfo: bktInfo, ObjectName: reqInfo.ObjectName, VersionID: reqInfo.URL.Query().Get(api.QueryVersionID), @@ -87,8 +85,8 @@ func (h *handler) GetObjectTaggingHandler(w http.ResponseWriter, r *http.Request return } - tagPrm := &layer.GetObjectTaggingParams{ - ObjectVersion: &layer.ObjectVersion{ + tagPrm := &data.GetObjectTaggingParams{ + ObjectVersion: &data.ObjectVersion{ BktInfo: bktInfo, ObjectName: reqInfo.ObjectName, VersionID: reqInfo.URL.Query().Get(api.QueryVersionID), @@ -119,7 +117,7 @@ func (h *handler) DeleteObjectTaggingHandler(w http.ResponseWriter, r *http.Requ return } - p := &layer.ObjectVersion{ + p := &data.ObjectVersion{ BktInfo: bktInfo, ObjectName: reqInfo.ObjectName, VersionID: reqInfo.URL.Query().Get(api.QueryVersionID), @@ -152,7 +150,7 @@ func (h *handler) DeleteObjectTaggingHandler(w http.ResponseWriter, r *http.Requ func (h *handler) PutBucketTaggingHandler(w http.ResponseWriter, r *http.Request) { reqInfo := middleware.GetReqInfo(r.Context()) - tagSet, err := h.readTagSet(r.Body) + tagSet, err := h.readTagSet(reqInfo.Tagging) if err != nil { h.logAndSendError(w, "could not read tag set", reqInfo, err) return @@ -207,12 +205,7 @@ func (h *handler) DeleteBucketTaggingHandler(w http.ResponseWriter, r *http.Requ w.WriteHeader(http.StatusNoContent) } -func (h *handler) readTagSet(reader io.Reader) (map[string]string, error) { - tagging := new(Tagging) - if err := h.cfg.NewXMLDecoder(reader).Decode(tagging); err != nil { - return nil, errors.GetAPIError(errors.ErrMalformedXML) - } - +func (h *handler) readTagSet(tagging *data.Tagging) (map[string]string, error) { if err := checkTagSet(tagging.TagSet); err != nil { return nil, err } @@ -228,10 +221,10 @@ func (h *handler) readTagSet(reader io.Reader) (map[string]string, error) { return tagSet, nil } -func encodeTagging(tagSet map[string]string) *Tagging { - tagging := &Tagging{} +func encodeTagging(tagSet map[string]string) *data.Tagging { + tagging := &data.Tagging{} for k, v := range tagSet { - tagging.TagSet = append(tagging.TagSet, Tag{Key: k, Value: v}) + tagging.TagSet = append(tagging.TagSet, data.Tag{Key: k, Value: v}) } sort.Slice(tagging.TagSet, func(i, j int) bool { return tagging.TagSet[i].Key < tagging.TagSet[j].Key @@ -240,7 +233,7 @@ func encodeTagging(tagSet map[string]string) *Tagging { return tagging } -func checkTagSet(tagSet []Tag) error { +func checkTagSet(tagSet []data.Tag) error { if len(tagSet) > maxTags { return errors.GetAPIError(errors.ErrInvalidTagsSizeExceed) } @@ -254,7 +247,7 @@ func checkTagSet(tagSet []Tag) error { return nil } -func checkTag(tag Tag) error { +func checkTag(tag data.Tag) error { if len(tag.Key) < 1 || len(tag.Key) > keyTagMaxLength { return errors.GetAPIError(errors.ErrInvalidTagKey) } diff --git a/api/handler/tagging_test.go b/api/handler/tagging_test.go index 27ef7be..5537aa6 100644 --- a/api/handler/tagging_test.go +++ b/api/handler/tagging_test.go @@ -5,7 +5,9 @@ import ( "strings" "testing" + "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data" apiErrors "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors" + "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/middleware" "github.com/stretchr/testify/require" ) @@ -20,23 +22,23 @@ func TestTagsValidity(t *testing.T) { } for _, tc := range []struct { - tag Tag + tag data.Tag valid bool }{ - {tag: Tag{}, valid: false}, - {tag: Tag{Key: "", Value: "1"}, valid: false}, - {tag: Tag{Key: "aws:key", Value: "val"}, valid: false}, - {tag: Tag{Key: "key~", Value: "val"}, valid: false}, - {tag: Tag{Key: "key\\", Value: "val"}, valid: false}, - {tag: Tag{Key: "key?", Value: "val"}, valid: false}, - {tag: Tag{Key: sbKey.String() + "b", Value: "val"}, valid: false}, - {tag: Tag{Key: "key", Value: sbValue.String() + "b"}, valid: false}, + {tag: data.Tag{}, valid: false}, + {tag: data.Tag{Key: "", Value: "1"}, valid: false}, + {tag: data.Tag{Key: "aws:key", Value: "val"}, valid: false}, + {tag: data.Tag{Key: "key~", Value: "val"}, valid: false}, + {tag: data.Tag{Key: "key\\", Value: "val"}, valid: false}, + {tag: data.Tag{Key: "key?", Value: "val"}, valid: false}, + {tag: data.Tag{Key: sbKey.String() + "b", Value: "val"}, valid: false}, + {tag: data.Tag{Key: "key", Value: sbValue.String() + "b"}, valid: false}, - {tag: Tag{Key: sbKey.String(), Value: "val"}, valid: true}, - {tag: Tag{Key: "key", Value: sbValue.String()}, valid: true}, - {tag: Tag{Key: "k e y", Value: "v a l"}, valid: true}, - {tag: Tag{Key: "12345", Value: "1234"}, valid: true}, - {tag: Tag{Key: allowedTagChars, Value: allowedTagChars}, valid: true}, + {tag: data.Tag{Key: sbKey.String(), Value: "val"}, valid: true}, + {tag: data.Tag{Key: "key", Value: sbValue.String()}, valid: true}, + {tag: data.Tag{Key: "k e y", Value: "v a l"}, valid: true}, + {tag: data.Tag{Key: "12345", Value: "1234"}, valid: true}, + {tag: data.Tag{Key: allowedTagChars, Value: allowedTagChars}, valid: true}, } { err := checkTag(tc.tag) if tc.valid { @@ -55,13 +57,13 @@ func TestPutObjectTaggingCheckUniqueness(t *testing.T) { for _, tc := range []struct { name string - body *Tagging + body *data.Tagging error bool }{ { name: "Two tags with unique keys", - body: &Tagging{ - TagSet: []Tag{ + body: &data.Tagging{ + TagSet: []data.Tag{ { Key: "key-1", Value: "val-1", @@ -76,8 +78,8 @@ func TestPutObjectTaggingCheckUniqueness(t *testing.T) { }, { name: "Two tags with the same keys", - body: &Tagging{ - TagSet: []Tag{ + body: &data.Tagging{ + TagSet: []data.Tag{ { Key: "key-1", Value: "val-1", @@ -93,6 +95,7 @@ func TestPutObjectTaggingCheckUniqueness(t *testing.T) { } { t.Run(tc.name, func(t *testing.T) { w, r := prepareTestRequest(hc, bktName, objName, tc.body) + middleware.GetReqInfo(r.Context()).Tagging = tc.body hc.Handler().PutObjectTaggingHandler(w, r) if tc.error { assertS3Error(t, w, apiErrors.GetAPIError(apiErrors.ErrInvalidTagKeyUniqueness)) diff --git a/api/layer/compound.go b/api/layer/compound.go index c17c95b..d4560d3 100644 --- a/api/layer/compound.go +++ b/api/layer/compound.go @@ -9,7 +9,7 @@ import ( s3errors "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors" ) -func (n *layer) GetObjectTaggingAndLock(ctx context.Context, objVersion *ObjectVersion, nodeVersion *data.NodeVersion) (map[string]string, data.LockInfo, error) { +func (n *layer) GetObjectTaggingAndLock(ctx context.Context, objVersion *data.ObjectVersion, nodeVersion *data.NodeVersion) (map[string]string, data.LockInfo, error) { var err error owner := n.BearerOwner(ctx) diff --git a/api/layer/layer.go b/api/layer/layer.go index a1d8282..a08540c 100644 --- a/api/layer/layer.go +++ b/api/layer/layer.go @@ -97,14 +97,6 @@ type ( VersionID string } - // ObjectVersion stores object version info. - ObjectVersion struct { - BktInfo *data.BucketInfo - ObjectName string - VersionID string - NoErrorOnDeleteMarker bool - } - // RangeParams stores range header request parameters. RangeParams struct { Start uint64 @@ -244,16 +236,16 @@ type ( GetObjectInfo(ctx context.Context, p *HeadObjectParams) (*data.ObjectInfo, error) GetExtendedObjectInfo(ctx context.Context, p *HeadObjectParams) (*data.ExtendedObjectInfo, error) - GetLockInfo(ctx context.Context, obj *ObjectVersion) (*data.LockInfo, error) + GetLockInfo(ctx context.Context, obj *data.ObjectVersion) (*data.LockInfo, error) PutLockInfo(ctx context.Context, p *PutLockInfoParams) error GetBucketTagging(ctx context.Context, bktInfo *data.BucketInfo) (map[string]string, error) PutBucketTagging(ctx context.Context, bktInfo *data.BucketInfo, tagSet map[string]string) error DeleteBucketTagging(ctx context.Context, bktInfo *data.BucketInfo) error - GetObjectTagging(ctx context.Context, p *GetObjectTaggingParams) (string, map[string]string, error) - PutObjectTagging(ctx context.Context, p *PutObjectTaggingParams) (*data.NodeVersion, error) - DeleteObjectTagging(ctx context.Context, p *ObjectVersion) (*data.NodeVersion, error) + GetObjectTagging(ctx context.Context, p *data.GetObjectTaggingParams) (string, map[string]string, error) + PutObjectTagging(ctx context.Context, p *data.PutObjectTaggingParams) (*data.NodeVersion, error) + DeleteObjectTagging(ctx context.Context, p *data.ObjectVersion) (*data.NodeVersion, error) PutObject(ctx context.Context, p *PutObjectParams) (*data.ExtendedObjectInfo, error) @@ -279,7 +271,7 @@ type ( // Compound methods for optimizations // GetObjectTaggingAndLock unifies GetObjectTagging and GetLock methods in single tree service invocation. - GetObjectTaggingAndLock(ctx context.Context, p *ObjectVersion, nodeVersion *data.NodeVersion) (map[string]string, data.LockInfo, error) + GetObjectTaggingAndLock(ctx context.Context, p *data.ObjectVersion, nodeVersion *data.NodeVersion) (map[string]string, data.LockInfo, error) } ) @@ -765,7 +757,7 @@ func isNotFoundError(err error) bool { } func (n *layer) getNodeVersionToDelete(ctx context.Context, bkt *data.BucketInfo, obj *VersionedObject) (*data.NodeVersion, error) { - objVersion := &ObjectVersion{ + objVersion := &data.ObjectVersion{ BktInfo: bkt, ObjectName: obj.Name, VersionID: obj.VersionID, @@ -776,7 +768,7 @@ func (n *layer) getNodeVersionToDelete(ctx context.Context, bkt *data.BucketInfo } func (n *layer) getLastNodeVersion(ctx context.Context, bkt *data.BucketInfo, obj *VersionedObject) (*data.NodeVersion, error) { - objVersion := &ObjectVersion{ + objVersion := &data.ObjectVersion{ BktInfo: bkt, ObjectName: obj.Name, VersionID: "", diff --git a/api/layer/locking_test.go b/api/layer/locking_test.go index 94ff1f2..d26c3be 100644 --- a/api/layer/locking_test.go +++ b/api/layer/locking_test.go @@ -20,7 +20,7 @@ func TestObjectLockAttributes(t *testing.T) { obj := tc.putObject([]byte("content obj1 v1")) p := &PutLockInfoParams{ - ObjVersion: &ObjectVersion{ + ObjVersion: &data.ObjectVersion{ BktInfo: tc.bktInfo, ObjectName: obj.Name, VersionID: obj.VersionID(), diff --git a/api/layer/object.go b/api/layer/object.go index e29582f..eb5412c 100644 --- a/api/layer/object.go +++ b/api/layer/object.go @@ -321,7 +321,7 @@ func (n *layer) PutObject(ctx context.Context, p *PutObjectParams) (*data.Extend if p.Lock != nil && (p.Lock.Retention != nil || p.Lock.LegalHold != nil) { putLockInfoPrms := &PutLockInfoParams{ - ObjVersion: &ObjectVersion{ + ObjVersion: &data.ObjectVersion{ BktInfo: p.BktInfo, ObjectName: p.Object, VersionID: id.EncodeToString(), diff --git a/api/layer/system_object.go b/api/layer/system_object.go index 0bfc455..1100072 100644 --- a/api/layer/system_object.go +++ b/api/layer/system_object.go @@ -20,7 +20,7 @@ const ( ) type PutLockInfoParams struct { - ObjVersion *ObjectVersion + ObjVersion *data.ObjectVersion NewLock *data.ObjectLock CopiesNumbers []uint32 NodeVersion *data.NodeVersion // optional @@ -100,7 +100,7 @@ func (n *layer) PutLockInfo(ctx context.Context, p *PutLockInfoParams) (err erro return nil } -func (n *layer) getNodeVersionFromCacheOrFrostfs(ctx context.Context, objVersion *ObjectVersion) (nodeVersion *data.NodeVersion, err error) { +func (n *layer) getNodeVersionFromCacheOrFrostfs(ctx context.Context, objVersion *data.ObjectVersion) (nodeVersion *data.NodeVersion, err error) { // check cache if node version is stored inside extendedObjectVersion nodeVersion = n.getNodeVersionFromCache(n.BearerOwner(ctx), objVersion) if nodeVersion == nil { @@ -129,7 +129,7 @@ func (n *layer) putLockObject(ctx context.Context, bktInfo *data.BucketInfo, obj return id, err } -func (n *layer) GetLockInfo(ctx context.Context, objVersion *ObjectVersion) (*data.LockInfo, error) { +func (n *layer) GetLockInfo(ctx context.Context, objVersion *data.ObjectVersion) (*data.LockInfo, error) { owner := n.BearerOwner(ctx) if lockInfo := n.cache.GetLockInfo(owner, lockObjectKey(objVersion)); lockInfo != nil { return lockInfo, nil @@ -185,7 +185,7 @@ func (n *layer) getCORS(ctx context.Context, bkt *data.BucketInfo) (*data.CORSCo return cors, nil } -func lockObjectKey(objVersion *ObjectVersion) string { +func lockObjectKey(objVersion *data.ObjectVersion) string { // todo reconsider forming name since versionID can be "null" or "" return ".lock." + objVersion.BktInfo.CID.EncodeToString() + "." + objVersion.ObjectName + "." + objVersion.VersionID } diff --git a/api/layer/tagging.go b/api/layer/tagging.go index 278d765..ba10653 100644 --- a/api/layer/tagging.go +++ b/api/layer/tagging.go @@ -14,22 +14,7 @@ import ( "go.uber.org/zap" ) -type GetObjectTaggingParams struct { - ObjectVersion *ObjectVersion - - // NodeVersion can be nil. If not nil we save one request to tree service. - NodeVersion *data.NodeVersion // optional -} - -type PutObjectTaggingParams struct { - ObjectVersion *ObjectVersion - TagSet map[string]string - - // NodeVersion can be nil. If not nil we save one request to tree service. - NodeVersion *data.NodeVersion // optional -} - -func (n *layer) GetObjectTagging(ctx context.Context, p *GetObjectTaggingParams) (string, map[string]string, error) { +func (n *layer) GetObjectTagging(ctx context.Context, p *data.GetObjectTaggingParams) (string, map[string]string, error) { var err error owner := n.BearerOwner(ctx) @@ -65,7 +50,7 @@ func (n *layer) GetObjectTagging(ctx context.Context, p *GetObjectTaggingParams) return p.ObjectVersion.VersionID, tags, nil } -func (n *layer) PutObjectTagging(ctx context.Context, p *PutObjectTaggingParams) (nodeVersion *data.NodeVersion, err error) { +func (n *layer) PutObjectTagging(ctx context.Context, p *data.PutObjectTaggingParams) (nodeVersion *data.NodeVersion, err error) { nodeVersion = p.NodeVersion if nodeVersion == nil { nodeVersion, err = n.getNodeVersionFromCacheOrFrostfs(ctx, p.ObjectVersion) @@ -88,7 +73,7 @@ func (n *layer) PutObjectTagging(ctx context.Context, p *PutObjectTaggingParams) return nodeVersion, nil } -func (n *layer) DeleteObjectTagging(ctx context.Context, p *ObjectVersion) (*data.NodeVersion, error) { +func (n *layer) DeleteObjectTagging(ctx context.Context, p *data.ObjectVersion) (*data.NodeVersion, error) { version, err := n.getNodeVersion(ctx, p) if err != nil { return nil, err @@ -142,7 +127,7 @@ func (n *layer) DeleteBucketTagging(ctx context.Context, bktInfo *data.BucketInf return n.treeService.DeleteBucketTagging(ctx, bktInfo) } -func objectTaggingCacheKey(p *ObjectVersion) string { +func objectTaggingCacheKey(p *data.ObjectVersion) string { return ".tagset." + p.BktInfo.CID.EncodeToString() + "." + p.ObjectName + "." + p.VersionID } @@ -150,7 +135,7 @@ func bucketTaggingCacheKey(cnrID cid.ID) string { return ".tagset." + cnrID.EncodeToString() } -func (n *layer) getNodeVersion(ctx context.Context, objVersion *ObjectVersion) (*data.NodeVersion, error) { +func (n *layer) getNodeVersion(ctx context.Context, objVersion *data.ObjectVersion) (*data.NodeVersion, error) { var err error var version *data.NodeVersion @@ -188,7 +173,7 @@ func (n *layer) getNodeVersion(ctx context.Context, objVersion *ObjectVersion) ( return version, err } -func (n *layer) getNodeVersionFromCache(owner user.ID, o *ObjectVersion) *data.NodeVersion { +func (n *layer) getNodeVersionFromCache(owner user.ID, o *data.ObjectVersion) *data.NodeVersion { if len(o.VersionID) == 0 || o.VersionID == data.UnversionedObjectVersionID { return nil } diff --git a/api/middleware/policy.go b/api/middleware/policy.go index 325dfb9..48f1eb3 100644 --- a/api/middleware/policy.go +++ b/api/middleware/policy.go @@ -3,12 +3,16 @@ package middleware import ( "context" "crypto/elliptic" + "encoding/xml" "fmt" + "io" "net/http" + "net/url" "strings" "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data" apiErr "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors" + frostfsErrors "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/frostfs/errors" "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs" "git.frostfs.info/TrueCloudLab/policy-engine/pkg/chain" "git.frostfs.info/TrueCloudLab/policy-engine/pkg/engine" @@ -25,8 +29,22 @@ const ( QueryPrefix = "prefix" QueryDelimiter = "delimiter" QueryMaxKeys = "max-keys" + amzTagging = "x-amz-tagging" ) +// At the beginning of these operations resources haven't yet been created. +var withoutResourceOps = []string{ + CreateBucketOperation, + CreateMultipartUploadOperation, + AbortMultipartUploadOperation, + CompleteMultipartUploadOperation, + UploadPartOperation, + UploadPartCopyOperation, + ListPartsOperation, + PutObjectOperation, + CopyObjectOperation, +} + type PolicySettings interface { PolicyDenyByDefault() bool ACLEnabled() bool @@ -36,6 +54,15 @@ type FrostFSIDInformer interface { GetUserGroupIDs(userHash util.Uint160) ([]string, error) } +type XMLDecoder interface { + NewXMLDecoder(io.Reader) *xml.Decoder +} + +type ResourceTagging interface { + GetBucketTagging(ctx context.Context, bktInfo *data.BucketInfo) (map[string]string, error) + GetObjectTagging(ctx context.Context, p *data.GetObjectTaggingParams) (string, map[string]string, error) +} + // BucketResolveFunc is a func to resolve bucket info by name. type BucketResolveFunc func(ctx context.Context, bucket string) (*data.BucketInfo, error) @@ -46,6 +73,8 @@ type PolicyConfig struct { Domains []string Log *zap.Logger BucketResolver BucketResolveFunc + Decoder XMLDecoder + Tagging ResourceTagging } func PolicyCheck(cfg PolicyConfig) Func { @@ -54,6 +83,7 @@ func PolicyCheck(cfg PolicyConfig) Func { ctx := r.Context() if err := policyCheck(r, cfg); err != nil { reqLogOrDefault(ctx, cfg.Log).Error(logs.PolicyValidationFailed, zap.Error(err)) + err = frostfsErrors.UnwrapErr(err) if _, wrErr := WriteErrorResponse(w, GetReqInfo(ctx), err); wrErr != nil { reqLogOrDefault(ctx, cfg.Log).Error(logs.FailedToWriteResponse, zap.Error(wrErr)) } @@ -67,7 +97,7 @@ func PolicyCheck(cfg PolicyConfig) Func { func policyCheck(r *http.Request, cfg PolicyConfig) error { reqType, bktName, objName := getBucketObject(r, cfg.Domains) - req, err := getPolicyRequest(r, cfg.FrostfsID, reqType, bktName, objName, cfg.Log) + req, err := getPolicyRequest(r, cfg, reqType, bktName, objName) if err != nil { return err } @@ -115,7 +145,7 @@ func policyCheck(r *http.Request, cfg PolicyConfig) error { return nil } -func getPolicyRequest(r *http.Request, frostfsid FrostFSIDInformer, reqType ReqType, bktName string, objName string, log *zap.Logger) (*testutil.Request, error) { +func getPolicyRequest(r *http.Request, cfg PolicyConfig, reqType ReqType, bktName string, objName string) (*testutil.Request, error) { var ( owner string groups []string @@ -130,7 +160,7 @@ func getPolicyRequest(r *http.Request, frostfsid FrostFSIDInformer, reqType ReqT } owner = pk.Address() - groups, err = frostfsid.GetUserGroupIDs(pk.GetScriptHash()) + groups, err = cfg.FrostfsID.GetUserGroupIDs(pk.GetScriptHash()) if err != nil { return nil, fmt.Errorf("get group ids: %w", err) } @@ -145,9 +175,12 @@ func getPolicyRequest(r *http.Request, frostfsid FrostFSIDInformer, reqType ReqT res = fmt.Sprintf(s3.ResourceFormatS3Bucket, bktName) } - properties := determineProperties(ctx, reqType, op, owner, groups) + properties, err := determineProperties(r, cfg.Decoder, cfg.BucketResolver, cfg.Tagging, reqType, op, bktName, objName, owner, groups) + if err != nil { + return nil, fmt.Errorf("determine properties: %w", err) + } - reqLogOrDefault(r.Context(), log).Debug(logs.PolicyRequest, zap.String("action", op), + reqLogOrDefault(r.Context(), cfg.Log).Debug(logs.PolicyRequest, zap.String("action", op), zap.String("resource", res), zap.Any("properties", properties)) return testutil.NewRequest(op, testutil.NewResource(res, nil), properties), nil @@ -376,12 +409,13 @@ func determineGeneralOperation(r *http.Request) string { return "UnmatchedOperation" } -func determineProperties(ctx context.Context, reqType ReqType, op, owner string, groups []string) map[string]string { +func determineProperties(r *http.Request, decoder XMLDecoder, resolver BucketResolveFunc, tagging ResourceTagging, reqType ReqType, + op, bktName, objName, owner string, groups []string) (map[string]string, error) { res := map[string]string{ s3.PropertyKeyOwner: owner, common.PropertyKeyFrostFSIDGroupID: chain.FormCondSliceContainsValue(groups), } - queries := GetReqInfo(ctx).URL.Query() + queries := GetReqInfo(r.Context()).URL.Query() if reqType == objectType { if versionID := queries.Get(QueryVersionID); len(versionID) > 0 { @@ -402,5 +436,107 @@ func determineProperties(ctx context.Context, reqType ReqType, op, owner string, } } - return res + tags, err := determineTags(r, decoder, resolver, tagging, reqType, op, bktName, objName, queries.Get(QueryVersionID)) + if err != nil { + return nil, fmt.Errorf("determine tags: %w", err) + } + for k, v := range tags { + res[k] = v + } + + return res, nil +} + +func determineTags(r *http.Request, decoder XMLDecoder, resolver BucketResolveFunc, tagging ResourceTagging, reqType ReqType, + op, bktName, objName, versionID string) (map[string]string, error) { + res, err := determineRequestTags(r, decoder, op) + if err != nil { + return nil, fmt.Errorf("determine request tags: %w", err) + } + + tags, err := determineResourceTags(r.Context(), reqType, op, bktName, objName, versionID, resolver, tagging) + if err != nil { + return nil, fmt.Errorf("determine resource tags: %w", err) + } + for k, v := range tags { + res[k] = v + } + + return res, nil +} + +func determineRequestTags(r *http.Request, decoder XMLDecoder, op string) (map[string]string, error) { + tags := make(map[string]string) + + if strings.HasSuffix(op, PutObjectTaggingOperation) || strings.HasSuffix(op, PutBucketTaggingOperation) { + tagging := new(data.Tagging) + if err := decoder.NewXMLDecoder(r.Body).Decode(tagging); err != nil { + return nil, apiErr.GetAPIError(apiErr.ErrMalformedXML) + } + GetReqInfo(r.Context()).Tagging = tagging + + for _, tag := range tagging.TagSet { + tags[fmt.Sprintf(s3.PropertyKeyFormatRequestTag, tag.Key)] = tag.Value + } + } + + if tagging := r.Header.Get(amzTagging); len(tagging) > 0 { + queries, err := url.ParseQuery(tagging) + if err != nil { + return nil, apiErr.GetAPIError(apiErr.ErrInvalidArgument) + } + for key := range queries { + tags[fmt.Sprintf(s3.PropertyKeyFormatRequestTag, key)] = queries.Get(key) + } + } + + return tags, nil +} + +func determineResourceTags(ctx context.Context, reqType ReqType, op, bktName, objName, versionID string, resolver BucketResolveFunc, + tagging ResourceTagging) (map[string]string, error) { + tags := make(map[string]string) + + if reqType != bucketType && reqType != objectType { + return tags, nil + } + + for _, withoutResOp := range withoutResourceOps { + if strings.HasSuffix(op, withoutResOp) { + return tags, nil + } + } + + bktInfo, err := resolver(ctx, bktName) + if err != nil { + return nil, fmt.Errorf("get bucket info: %w", err) + } + + if reqType == bucketType { + tags, err = tagging.GetBucketTagging(ctx, bktInfo) + if err != nil { + return nil, fmt.Errorf("get bucket tagging: %w", err) + } + } + + if reqType == objectType { + tagPrm := &data.GetObjectTaggingParams{ + ObjectVersion: &data.ObjectVersion{ + BktInfo: bktInfo, + ObjectName: objName, + VersionID: versionID, + }, + } + _, tags, err = tagging.GetObjectTagging(ctx, tagPrm) + if err != nil { + return nil, fmt.Errorf("get object tagging: %w", err) + } + } + + res := make(map[string]string, len(tags)) + for k, v := range tags { + res[fmt.Sprintf(s3.PropertyKeyFormatResourceTag, k)] = v + } + + return res, nil } diff --git a/api/middleware/reqinfo.go b/api/middleware/reqinfo.go index 9e55471..2b071c9 100644 --- a/api/middleware/reqinfo.go +++ b/api/middleware/reqinfo.go @@ -39,7 +39,8 @@ type ( TraceID string // Trace ID URL *url.URL // Request url Namespace string - User string // User owner id + User string // User owner id + Tagging *data.Tagging tags []KeyVal // Any additional info not accommodated by above fields } diff --git a/api/router.go b/api/router.go index 86e0362..a6dfa0a 100644 --- a/api/router.go +++ b/api/router.go @@ -121,6 +121,9 @@ type Config struct { FrostFSIDValidation bool PolicyChecker engine.ChainRouter + + XMLDecoder s3middleware.XMLDecoder + Tagging s3middleware.ResourceTagging } func NewRouter(cfg Config) *chi.Mux { @@ -146,6 +149,8 @@ func NewRouter(cfg Config) *chi.Mux { Domains: cfg.Domains, Log: cfg.Log, BucketResolver: cfg.Handler.ResolveBucket, + Decoder: cfg.XMLDecoder, + Tagging: cfg.Tagging, })) defaultRouter := chi.NewRouter() diff --git a/api/router_mock_test.go b/api/router_mock_test.go index e8fe457..8747b1a 100644 --- a/api/router_mock_test.go +++ b/api/router_mock_test.go @@ -3,11 +3,13 @@ package api import ( "context" "encoding/json" - "errors" + "encoding/xml" + "io" "net/http" "testing" "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data" + apiErrors "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors" "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/middleware" "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/creds/accessbox" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer" @@ -86,6 +88,30 @@ func (f *frostFSIDMock) GetUserGroupIDs(util.Uint160) ([]string, error) { return []string{}, nil } +type xmlMock struct { +} + +func (m *xmlMock) NewXMLDecoder(r io.Reader) *xml.Decoder { + return xml.NewDecoder(r) +} + +type resourceTaggingMock struct { + bucketTags map[string]string + objectTags map[string]string + noSuchKey bool +} + +func (m *resourceTaggingMock) GetBucketTagging(context.Context, *data.BucketInfo) (map[string]string, error) { + return m.bucketTags, nil +} + +func (m *resourceTaggingMock) GetObjectTagging(context.Context, *data.GetObjectTaggingParams) (string, map[string]string, error) { + if m.noSuchKey { + return "", nil, apiErrors.GetAPIError(apiErrors.ErrNoSuchKey) + } + return "", m.objectTags, nil +} + type handlerMock struct { t *testing.T cfg *middlewareSettingsMock @@ -142,9 +168,13 @@ func (h *handlerMock) GetObjectLegalHoldHandler(http.ResponseWriter, *http.Reque panic("implement me") } -func (h *handlerMock) GetObjectHandler(http.ResponseWriter, *http.Request) { - //TODO implement me - panic("implement me") +func (h *handlerMock) GetObjectHandler(w http.ResponseWriter, r *http.Request) { + res := &handlerResult{ + Method: middleware.GetObjectOperation, + ReqInfo: middleware.GetReqInfo(r.Context()), + } + + h.writeResponse(w, res) } func (h *handlerMock) GetObjectAttributesHandler(http.ResponseWriter, *http.Request) { @@ -339,9 +369,13 @@ func (h *handlerMock) PutBucketObjectLockConfigHandler(http.ResponseWriter, *htt panic("implement me") } -func (h *handlerMock) PutBucketTaggingHandler(http.ResponseWriter, *http.Request) { - //TODO implement me - panic("implement me") +func (h *handlerMock) PutBucketTaggingHandler(w http.ResponseWriter, r *http.Request) { + res := &handlerResult{ + Method: middleware.PutBucketTaggingOperation, + ReqInfo: middleware.GetReqInfo(r.Context()), + } + + h.writeResponse(w, res) } func (h *handlerMock) PutBucketVersioningHandler(http.ResponseWriter, *http.Request) { @@ -473,7 +507,7 @@ func (h *handlerMock) ResolveBucket(ctx context.Context, name string) (*data.Buc reqInfo := middleware.GetReqInfo(ctx) bktInfo, ok := h.buckets[reqInfo.Namespace+name] if !ok { - return nil, errors.New("not found") + return nil, apiErrors.GetAPIError(apiErrors.ErrNoSuchBucket) } return bktInfo, nil } diff --git a/api/router_test.go b/api/router_test.go index c272208..a401196 100644 --- a/api/router_test.go +++ b/api/router_test.go @@ -1,6 +1,7 @@ package api import ( + "bytes" "encoding/json" "encoding/xml" "fmt" @@ -66,6 +67,8 @@ func prepareRouter(t *testing.T) *routerMock { PolicyChecker: policyChecker, Domains: []string{"domain1", "domain2"}, FrostfsID: &frostFSIDMock{}, + XMLDecoder: &xmlMock{}, + Tagging: &resourceTaggingMock{}, } return &routerMock{ t: t, @@ -115,7 +118,7 @@ func TestRouterObjectWithSlashes(t *testing.T) { ns, bktName, objName := "", "dkirillov", "/fix/object" createBucket(chiRouter, ns, bktName) - resp := putObject(chiRouter, ns, bktName, objName) + resp := putObject(chiRouter, ns, bktName, objName, nil) require.Equal(t, objName, resp.ReqInfo.ObjectName) } @@ -157,7 +160,7 @@ func TestRouterObjectEscaping(t *testing.T) { }, } { t.Run(tc.name, func(t *testing.T) { - resp := putObject(chiRouter, ns, bktName, tc.objName) + resp := putObject(chiRouter, ns, bktName, tc.objName, nil) require.Equal(t, tc.expectedObjName, resp.ReqInfo.ObjectName) }) } @@ -185,13 +188,13 @@ func TestPolicyChecker(t *testing.T) { require.NoError(t, err) // check we can access 'bucket' in default namespace - putObject(chiRouter, ns1, bktName1, objName1) + putObject(chiRouter, ns1, bktName1, objName1, nil) // check we can access 'other-bucket' in custom namespace - putObject(chiRouter, ns2, bktName2, objName2) + putObject(chiRouter, ns2, bktName2, objName2, nil) // check we cannot access 'bucket' in custom namespace - putObjectErr(chiRouter, ns2, bktName1, objName2, apiErrors.ErrAccessDenied) + putObjectErr(chiRouter, ns2, bktName1, objName2, nil, apiErrors.ErrAccessDenied) } func TestPolicyCheckerReqTypeDetermination(t *testing.T) { @@ -215,6 +218,8 @@ func TestPolicyCheckerReqTypeDetermination(t *testing.T) { _, _, err = chiRouter.policyChecker.MorphRuleChainStorage().AddMorphRuleChain(chain.S3, engine.NamespaceTarget(""), ruleChain) require.NoError(t, err) + createBucket(chiRouter, "", bktName) + chiRouter.middlewareSettings.denyByDefault = true t.Run("can list buckets", func(t *testing.T) { w, r := httptest.NewRecorder(), httptest.NewRequest(http.MethodGet, "/", nil) @@ -263,9 +268,9 @@ func TestACLAPE(t *testing.T) { router.middlewareSettings.denyByDefault = true // Allow because of using old bucket - putObject(router, ns, bktNameOld, objName) + putObject(router, ns, bktNameOld, objName, nil) // Deny because of deny by default - putObjectErr(router, ns, bktNameNew, objName, apiErrors.ErrAccessDenied) + putObjectErr(router, ns, bktNameNew, objName, nil, apiErrors.ErrAccessDenied) // Deny because of deny by default createBucketErr(router, ns, bktName, apiErrors.ErrAccessDenied) @@ -289,9 +294,9 @@ func TestACLAPE(t *testing.T) { router.middlewareSettings.denyByDefault = false // Allow because of using old bucket - putObject(router, ns, bktNameOld, objName) + putObject(router, ns, bktNameOld, objName, nil) // Allow because of allow by default - putObject(router, ns, bktNameNew, objName) + putObject(router, ns, bktNameNew, objName, nil) // Allow because of deny by default createBucket(router, ns, bktName) @@ -315,9 +320,9 @@ func TestACLAPE(t *testing.T) { router.middlewareSettings.denyByDefault = true // Allow because of using old bucket - putObject(router, ns, bktNameOld, objName) + putObject(router, ns, bktNameOld, objName, nil) // Deny because of deny by default - putObjectErr(router, ns, bktNameNew, objName, apiErrors.ErrAccessDenied) + putObjectErr(router, ns, bktNameNew, objName, nil, apiErrors.ErrAccessDenied) // Allow because of old behavior createBucket(router, ns, bktName) @@ -336,9 +341,9 @@ func TestACLAPE(t *testing.T) { router.middlewareSettings.denyByDefault = false // Allow because of using old bucket - putObject(router, ns, bktNameOld, objName) + putObject(router, ns, bktNameOld, objName, nil) // Allow because of allow by default - putObject(router, ns, bktNameNew, objName) + putObject(router, ns, bktNameNew, objName, nil) // Allow because of old behavior createBucket(router, ns, bktName) @@ -459,6 +464,118 @@ func TestRequestParametersCheck(t *testing.T) { }) } +func TestRequestTagsCheck(t *testing.T) { + t.Run("put bucket tagging", func(t *testing.T) { + router := prepareRouter(t) + + ns, bktName, tagKey, tagValue := "", "bucket", "tag", "value" + router.middlewareSettings.denyByDefault = true + + allowOperations(router, ns, []string{"s3:CreateBucket"}, nil) + createBucket(router, ns, bktName) + + // Add policies and check + allowOperations(router, ns, []string{"s3:PutBucketTagging"}, engineiam.Conditions{ + engineiam.CondStringEquals: engineiam.Condition{fmt.Sprintf(s3.PropertyKeyFormatRequestTag, tagKey): []string{tagValue}}, + }) + denyOperations(router, ns, []string{"s3:PutBucketTagging"}, engineiam.Conditions{ + engineiam.CondStringNotEquals: engineiam.Condition{fmt.Sprintf(s3.PropertyKeyFormatRequestTag, tagKey): []string{tagValue}}, + }) + + tagging, err := xml.Marshal(data.Tagging{TagSet: []data.Tag{{Key: tagKey, Value: tagValue}}}) + require.NoError(t, err) + putBucketTagging(router, ns, bktName, tagging) + + tagging, err = xml.Marshal(data.Tagging{TagSet: []data.Tag{{Key: "key", Value: tagValue}}}) + require.NoError(t, err) + putBucketTaggingErr(router, ns, bktName, tagging, apiErrors.ErrAccessDenied) + }) + + t.Run("put object with tag", func(t *testing.T) { + router := prepareRouter(t) + + ns, bktName, objName, tagKey, tagValue := "", "bucket", "object", "tag", "value" + router.middlewareSettings.denyByDefault = true + + allowOperations(router, ns, []string{"s3:CreateBucket"}, nil) + createBucket(router, ns, bktName) + + // Add policies and check + allowOperations(router, ns, []string{"s3:PutObject"}, engineiam.Conditions{ + engineiam.CondStringEquals: engineiam.Condition{fmt.Sprintf(s3.PropertyKeyFormatRequestTag, tagKey): []string{tagValue}}, + }) + denyOperations(router, ns, []string{"s3:PutObject"}, engineiam.Conditions{ + engineiam.CondStringNotEquals: engineiam.Condition{fmt.Sprintf(s3.PropertyKeyFormatRequestTag, tagKey): []string{tagValue}}, + }) + + putObject(router, ns, bktName, objName, &data.Tag{Key: tagKey, Value: tagValue}) + + putObjectErr(router, ns, bktName, objName, &data.Tag{Key: "key", Value: tagValue}, apiErrors.ErrAccessDenied) + }) +} + +func TestResourceTagsCheck(t *testing.T) { + t.Run("bucket tagging", func(t *testing.T) { + router := prepareRouter(t) + + ns, bktName, tagKey, tagValue := "", "bucket", "tag", "value" + router.middlewareSettings.denyByDefault = true + + allowOperations(router, ns, []string{"s3:CreateBucket"}, nil) + createBucket(router, ns, bktName) + + // Add policies and check + allowOperations(router, ns, []string{"s3:ListBucket"}, engineiam.Conditions{ + engineiam.CondStringEquals: engineiam.Condition{fmt.Sprintf(s3.PropertyKeyFormatResourceTag, tagKey): []string{tagValue}}, + }) + denyOperations(router, ns, []string{"s3:ListBucket"}, engineiam.Conditions{ + engineiam.CondStringNotEquals: engineiam.Condition{fmt.Sprintf(s3.PropertyKeyFormatResourceTag, tagKey): []string{tagValue}}, + }) + + router.cfg.Tagging.(*resourceTaggingMock).bucketTags = map[string]string{tagKey: tagValue} + listObjectsV1(router, ns, bktName, "", "", "") + + router.cfg.Tagging.(*resourceTaggingMock).bucketTags = map[string]string{} + listObjectsV1Err(router, ns, bktName, "", "", "", apiErrors.ErrAccessDenied) + }) + + t.Run("object tagging", func(t *testing.T) { + router := prepareRouter(t) + + ns, bktName, objName, tagKey, tagValue := "", "bucket", "object", "tag", "value" + router.middlewareSettings.denyByDefault = true + + allowOperations(router, ns, []string{"s3:CreateBucket", "s3:PutObject"}, nil) + createBucket(router, ns, bktName) + putObject(router, ns, bktName, objName, nil) + + // Add policies and check + allowOperations(router, ns, []string{"s3:GetObject"}, engineiam.Conditions{ + engineiam.CondStringEquals: engineiam.Condition{fmt.Sprintf(s3.PropertyKeyFormatResourceTag, tagKey): []string{tagValue}}, + }) + denyOperations(router, ns, []string{"s3:GetObject"}, engineiam.Conditions{ + engineiam.CondStringNotEquals: engineiam.Condition{fmt.Sprintf(s3.PropertyKeyFormatResourceTag, tagKey): []string{tagValue}}, + }) + + router.cfg.Tagging.(*resourceTaggingMock).objectTags = map[string]string{tagKey: tagValue} + getObject(router, ns, bktName, objName) + + router.cfg.Tagging.(*resourceTaggingMock).objectTags = map[string]string{} + getObjectErr(router, ns, bktName, objName, apiErrors.ErrAccessDenied) + }) + + t.Run("non-existent resources", func(t *testing.T) { + router := prepareRouter(t) + ns, bktName, objName := "", "bucket", "object" + + listObjectsV1Err(router, ns, bktName, "", "", "", apiErrors.ErrNoSuchBucket) + + router.cfg.Tagging.(*resourceTaggingMock).noSuchKey = true + createBucket(router, ns, bktName) + getObjectErr(router, ns, bktName, objName, apiErrors.ErrNoSuchKey) + }) +} + func allowOperations(router *routerMock, ns string, operations []string, conditions engineiam.Conditions) { addPolicy(router, ns, "allow", engineiam.AllowEffect, operations, conditions) } @@ -538,20 +655,68 @@ func listBucketsBase(router *routerMock, namespace string) *httptest.ResponseRec return w } -func putObject(router *routerMock, namespace, bktName, objName string) handlerResult { - w := putObjectBase(router, namespace, bktName, objName) +func putObject(router *routerMock, namespace, bktName, objName string, tag *data.Tag) handlerResult { + w := putObjectBase(router, namespace, bktName, objName, tag) resp := readResponse(router.t, w) require.Equal(router.t, s3middleware.PutObjectOperation, resp.Method) return resp } -func putObjectErr(router *routerMock, namespace, bktName, objName string, errCode apiErrors.ErrorCode) { - w := putObjectBase(router, namespace, bktName, objName) +func putObjectErr(router *routerMock, namespace, bktName, objName string, tag *data.Tag, errCode apiErrors.ErrorCode) { + w := putObjectBase(router, namespace, bktName, objName, tag) assertAPIError(router.t, w, errCode) } -func putObjectBase(router *routerMock, namespace, bktName, objName string) *httptest.ResponseRecorder { +func putObjectBase(router *routerMock, namespace, bktName, objName string, tag *data.Tag) *httptest.ResponseRecorder { w, r := httptest.NewRecorder(), httptest.NewRequest(http.MethodPut, "/"+bktName+"/"+objName, nil) + if tag != nil { + queries := url.Values{ + tag.Key: []string{tag.Value}, + } + r.Header.Set(AmzTagging, queries.Encode()) + } + r.Header.Set(FrostfsNamespaceHeader, namespace) + router.ServeHTTP(w, r) + return w +} + +func putBucketTagging(router *routerMock, namespace, bktName string, tagging []byte) handlerResult { + w := putBucketTaggingBase(router, namespace, bktName, tagging) + resp := readResponse(router.t, w) + require.Equal(router.t, s3middleware.PutBucketTaggingOperation, resp.Method) + return resp +} + +func putBucketTaggingErr(router *routerMock, namespace, bktName string, tagging []byte, errCode apiErrors.ErrorCode) { + w := putBucketTaggingBase(router, namespace, bktName, tagging) + assertAPIError(router.t, w, errCode) +} + +func putBucketTaggingBase(router *routerMock, namespace, bktName string, tagging []byte) *httptest.ResponseRecorder { + queries := url.Values{} + queries.Add(s3middleware.TaggingQuery, "") + + w, r := httptest.NewRecorder(), httptest.NewRequest(http.MethodPut, "/"+bktName, bytes.NewBuffer(tagging)) + r.URL.RawQuery = queries.Encode() + r.Header.Set(FrostfsNamespaceHeader, namespace) + router.ServeHTTP(w, r) + return w +} + +func getObject(router *routerMock, namespace, bktName, objName string) handlerResult { + w := getObjectBase(router, namespace, bktName, objName) + resp := readResponse(router.t, w) + require.Equal(router.t, s3middleware.GetObjectOperation, resp.Method) + return resp +} + +func getObjectErr(router *routerMock, namespace, bktName, objName string, errCode apiErrors.ErrorCode) { + w := getObjectBase(router, namespace, bktName, objName) + assertAPIError(router.t, w, errCode) +} + +func getObjectBase(router *routerMock, namespace, bktName, objName string) *httptest.ResponseRecorder { + w, r := httptest.NewRecorder(), httptest.NewRequest(http.MethodGet, "/"+bktName+"/"+objName, nil) r.Header.Set(FrostfsNamespaceHeader, namespace) router.ServeHTTP(w, r) return w @@ -596,11 +761,11 @@ func TestOwnerIDRetrieving(t *testing.T) { createBucket(chiRouter, ns, bktName) - resp := putObject(chiRouter, ns, bktName, objName) + resp := putObject(chiRouter, ns, bktName, objName, nil) require.NotEqual(t, "anon", resp.ReqInfo.User) chiRouter.cfg.Center.(*centerMock).anon = true - resp = putObject(chiRouter, ns, bktName, objName) + resp = putObject(chiRouter, ns, bktName, objName, nil) require.Equal(t, "anon", resp.ReqInfo.User) } @@ -618,7 +783,7 @@ func TestBillingMetrics(t *testing.T) { require.Equal(t, 1, dump.Requests[0].Requests) chiRouter.cfg.Center.(*centerMock).anon = true - putObject(chiRouter, ns, bktName, objName) + putObject(chiRouter, ns, bktName, objName, nil) dump = chiRouter.cfg.Metrics.UsersAPIStats().DumpMetrics() require.Len(t, dump.Requests, 1) require.Equal(t, "anon", dump.Requests[0].User) diff --git a/cmd/s3-gw/app.go b/cmd/s3-gw/app.go index 7d4e8b6..af7e35a 100644 --- a/cmd/s3-gw/app.go +++ b/cmd/s3-gw/app.go @@ -688,6 +688,9 @@ func (a *App) Serve(ctx context.Context) { FrostfsID: a.frostfsid, FrostFSIDValidation: a.settings.frostfsidValidation, + + XMLDecoder: a.settings, + Tagging: a.obj, } chiRouter := api.NewRouter(cfg)