TrueCloudLab/frostfs-s3-gw
17
3
Fork 19
Code Issues 28 Pull requests 7 Projects Releases 17 Packages Activity Actions

[#357] Add check of request and resource tags #357

Merged
alexvanin merged 1 commit from mbiryukova/frostfs-s3-gw:feature/tags_to_policy_check into master 2024-04-17 07:07:00 +00:00
Conversation 0 Commits 1 Files changed 24 +506 -155
24 changed files with 506 additions and 155 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

8
api/data/info.go
View file

@ -83,6 +83,14 @@ type (
ExposeHeaders []string `xml:"ExposeHeader" json:"ExposeHeaders"`
MaxAgeSeconds int `xml:"MaxAgeSeconds,omitempty" json:"MaxAgeSeconds,omitempty"`
}
// ObjectVersion stores object version info.
ObjectVersion struct {
BktInfo *BucketInfo
ObjectName string
VersionID string
NoErrorOnDeleteMarker bool
}
)
// NotificationInfoFromObject creates new NotificationInfo from ObjectInfo.

30
api/data/tagging.go Normal file
View file

@ -0,0 +1,30 @@
package data
import "encoding/xml"
// Tagging contains tag set.
type Tagging struct {
XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Tagging"`
TagSet []Tag `xml:"TagSet>Tag"`
}
// Tag is an AWS key-value tag.
type Tag struct {
Key string
Value string
}
type GetObjectTaggingParams struct {
ObjectVersion *ObjectVersion
// NodeVersion can be nil. If not nil we save one request to tree service.
NodeVersion *NodeVersion // optional
}
type PutObjectTaggingParams struct {
ObjectVersion *ObjectVersion
TagSet map[string]string
// NodeVersion can be nil. If not nil we save one request to tree service.
NodeVersion *NodeVersion // optional
}

8
api/handler/copy.go
View file

@ -168,8 +168,8 @@ func (h *handler) CopyObjectHandler(w http.ResponseWriter, r *http.Request) {
return
}
} else {
tagPrm := &layer.GetObjectTaggingParams{
ObjectVersion: &layer.ObjectVersion{
tagPrm := &data.GetObjectTaggingParams{
ObjectVersion: &data.ObjectVersion{
BktInfo: srcObjPrm.BktInfo,
ObjectName: srcObject,
VersionID: srcObjInfo.VersionID(),
@ -259,8 +259,8 @@ func (h *handler) CopyObjectHandler(w http.ResponseWriter, r *http.Request) {
}
if tagSet != nil {
tagPrm := &layer.PutObjectTaggingParams{
ObjectVersion: &layer.ObjectVersion{
tagPrm := &data.PutObjectTaggingParams{
ObjectVersion: &data.ObjectVersion{
BktInfo: dstBktInfo,
ObjectName: reqInfo.ObjectName,
VersionID: dstObjInfo.VersionID(),

13
api/handler/copy_test.go
View file

@ -11,9 +11,11 @@ import (
"testing"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer/encryption"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/middleware"
"github.com/stretchr/testify/require"
)
@ -289,23 +291,24 @@ func copyObject(hc *handlerContext, bktName, fromObject, toObject string, copyMe
}
func putObjectTagging(t *testing.T, tc *handlerContext, bktName, objName string, tags map[string]string) {
body := &Tagging{
TagSet: make([]Tag, 0, len(tags)),
body := &data.Tagging{
TagSet: make([]data.Tag, 0, len(tags)),
}
for key, val := range tags {
body.TagSet = append(body.TagSet, Tag{
body.TagSet = append(body.TagSet, data.Tag{
Key: key,
Value: val,
})
}
w, r := prepareTestRequest(tc, bktName, objName, body)
middleware.GetReqInfo(r.Context()).Tagging = body
tc.Handler().PutObjectTaggingHandler(w, r)
assertStatus(t, w, http.StatusOK)
}
func getObjectTagging(t *testing.T, tc *handlerContext, bktName, objName, version string) *Tagging {
func getObjectTagging(t *testing.T, tc *handlerContext, bktName, objName, version string) *data.Tagging {
query := make(url.Values)
query.Add(api.QueryVersionID, version)
@ -313,7 +316,7 @@ func getObjectTagging(t *testing.T, tc *handlerContext, bktName, objName, versio
tc.Handler().GetObjectTaggingHandler(w, r)
assertStatus(t, w, http.StatusOK)
tagging := &Tagging{}
tagging := &data.Tagging{}
err := xml.NewDecoder(w.Result().Body).Decode(tagging)
require.NoError(t, err)
return tagging

2
api/handler/get.go
View file

@ -184,7 +184,7 @@ func (h *handler) GetObjectHandler(w http.ResponseWriter, r *http.Request) {
return
}
t := &layer.ObjectVersion{
t := &data.ObjectVersion{
BktInfo: bktInfo,
ObjectName: info.Name,
VersionID: info.VersionID(),

2
api/handler/head.go
View file

@ -70,7 +70,7 @@ func (h *handler) HeadObjectHandler(w http.ResponseWriter, r *http.Request) {
return
}
t := &layer.ObjectVersion{
t := &data.ObjectVersion{
BktInfo: bktInfo,
ObjectName: info.Name,
VersionID: info.VersionID(),

8
api/handler/locking.go
View file

@ -133,7 +133,7 @@ func (h *handler) PutObjectLegalHoldHandler(w http.ResponseWriter, r *http.Reque
}
p := &layer.PutLockInfoParams{
ObjVersion: &layer.ObjectVersion{
ObjVersion: &data.ObjectVersion{
BktInfo: bktInfo,
ObjectName: reqInfo.ObjectName,
VersionID: reqInfo.URL.Query().Get(api.QueryVersionID),
@ -172,7 +172,7 @@ func (h *handler) GetObjectLegalHoldHandler(w http.ResponseWriter, r *http.Reque
return
}
p := &layer.ObjectVersion{
p := &data.ObjectVersion{
BktInfo: bktInfo,
ObjectName: reqInfo.ObjectName,
VersionID: reqInfo.URL.Query().Get(api.QueryVersionID),
@ -221,7 +221,7 @@ func (h *handler) PutObjectRetentionHandler(w http.ResponseWriter, r *http.Reque
}
p := &layer.PutLockInfoParams{
ObjVersion: &layer.ObjectVersion{
ObjVersion: &data.ObjectVersion{
BktInfo: bktInfo,
ObjectName: reqInfo.ObjectName,
VersionID: reqInfo.URL.Query().Get(api.QueryVersionID),
@ -256,7 +256,7 @@ func (h *handler) GetObjectRetentionHandler(w http.ResponseWriter, r *http.Reque
return
}
p := &layer.ObjectVersion{
p := &data.ObjectVersion{
BktInfo: bktInfo,
ObjectName: reqInfo.ObjectName,
VersionID: reqInfo.URL.Query().Get(api.QueryVersionID),

4
api/handler/multipart_upload.go
View file

@ -487,8 +487,8 @@ func (h *handler) completeMultipartUpload(r *http.Request, c *layer.CompleteMult
objInfo := extendedObjInfo.ObjectInfo
if len(uploadData.TagSet) != 0 {
tagPrm := &layer.PutObjectTaggingParams{
ObjectVersion: &layer.ObjectVersion{
tagPrm := &data.PutObjectTaggingParams{
ObjectVersion: &data.ObjectVersion{
BktInfo: bktInfo,
ObjectName: objInfo.Name,
VersionID: objInfo.VersionID(),

17
api/handler/put.go
View file

@ -307,8 +307,8 @@ func (h *handler) PutObjectHandler(w http.ResponseWriter, r *http.Request) {
}
if tagSet != nil {
tagPrm := &layer.PutObjectTaggingParams{
ObjectVersion: &layer.ObjectVersion{
tagPrm := &data.PutObjectTaggingParams{
ObjectVersion: &data.ObjectVersion{
BktInfo: bktInfo,
ObjectName: objInfo.Name,
VersionID: objInfo.VersionID(),
@ -483,7 +483,12 @@ func (h *handler) PostObject(w http.ResponseWriter, r *http.Request) {
if tagging := auth.MultipartFormValue(r, "tagging"); tagging != "" {
buffer := bytes.NewBufferString(tagging)
tagSet, err = h.readTagSet(buffer)
tags := new(data.Tagging)
if err = h.cfg.NewXMLDecoder(buffer).Decode(tags); err != nil {
h.logAndSendError(w, "could not decode tag set", reqInfo, errors.GetAPIError(errors.ErrMalformedXML))
return
}
tagSet, err = h.readTagSet(tags)
if err != nil {
h.logAndSendError(w, "could not read tag set", reqInfo, err)
return
@ -574,8 +579,8 @@ func (h *handler) PostObject(w http.ResponseWriter, r *http.Request) {
}
if tagSet != nil {
tagPrm := &layer.PutObjectTaggingParams{
ObjectVersion: &layer.ObjectVersion{
tagPrm := &data.PutObjectTaggingParams{
ObjectVersion: &data.ObjectVersion{
BktInfo: bktInfo,
ObjectName: objInfo.Name,
VersionID: objInfo.VersionID(),
@ -789,7 +794,7 @@ func parseTaggingHeader(header http.Header) (map[string]string, error) {
}
tagSet = make(map[string]string, len(queries))
for k, v := range queries {
tag := Tag{Key: k, Value: v[0]}
tag := data.Tag{Key: k, Value: v[0]}
if err = checkTag(tag); err != nil {
return nil, err
}

12
api/handler/response.go
View file

@ -185,12 +185,6 @@ type VersioningConfiguration struct {
MfaDelete string `xml:"MfaDelete,omitempty"`
}
// Tagging contains tag set.
type Tagging struct {
XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Tagging"`
TagSet []Tag `xml:"TagSet>Tag"`
}
// PostResponse contains result of posting object.
type PostResponse struct {
Bucket string `xml:"Bucket"`
@ -198,12 +192,6 @@ type PostResponse struct {
ETag string `xml:"Etag"`
}
// Tag is an AWS key-value tag.
type Tag struct {
Key string
Value string
}
// MarshalXML -- StringMap marshals into XML.
func (s StringMap) MarshalXML(e *xml.Encoder, start xml.StartElement) error {
tokens := []xml.Token{start}

33
api/handler/tagging.go
View file

@ -1,7 +1,6 @@
package handler
import (
"io"
"net/http"
"sort"
"strings"
@ -10,7 +9,6 @@ import (
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/middleware"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs"
"go.uber.org/zap"
@ -28,7 +26,7 @@ func (h *handler) PutObjectTaggingHandler(w http.ResponseWriter, r *http.Request
ctx := r.Context()
reqInfo := middleware.GetReqInfo(ctx)
tagSet, err := h.readTagSet(r.Body)
tagSet, err := h.readTagSet(reqInfo.Tagging)
if err != nil {
h.logAndSendError(w, "could not read tag set", reqInfo, err)
return
@ -40,8 +38,8 @@ func (h *handler) PutObjectTaggingHandler(w http.ResponseWriter, r *http.Request
return
}
tagPrm := &layer.PutObjectTaggingParams{
ObjectVersion: &layer.ObjectVersion{
tagPrm := &data.PutObjectTaggingParams{
ObjectVersion: &data.ObjectVersion{
BktInfo: bktInfo,
ObjectName: reqInfo.ObjectName,
VersionID: reqInfo.URL.Query().Get(api.QueryVersionID),
@ -87,8 +85,8 @@ func (h *handler) GetObjectTaggingHandler(w http.ResponseWriter, r *http.Request
return
}
tagPrm := &layer.GetObjectTaggingParams{
ObjectVersion: &layer.ObjectVersion{
tagPrm := &data.GetObjectTaggingParams{
ObjectVersion: &data.ObjectVersion{
BktInfo: bktInfo,
ObjectName: reqInfo.ObjectName,
VersionID: reqInfo.URL.Query().Get(api.QueryVersionID),
@ -119,7 +117,7 @@ func (h *handler) DeleteObjectTaggingHandler(w http.ResponseWriter, r *http.Requ
return
}
p := &layer.ObjectVersion{
p := &data.ObjectVersion{
BktInfo: bktInfo,
ObjectName: reqInfo.ObjectName,
VersionID: reqInfo.URL.Query().Get(api.QueryVersionID),
@ -152,7 +150,7 @@ func (h *handler) DeleteObjectTaggingHandler(w http.ResponseWriter, r *http.Requ
func (h *handler) PutBucketTaggingHandler(w http.ResponseWriter, r *http.Request) {
reqInfo := middleware.GetReqInfo(r.Context())
tagSet, err := h.readTagSet(r.Body)
tagSet, err := h.readTagSet(reqInfo.Tagging)
if err != nil {
h.logAndSendError(w, "could not read tag set", reqInfo, err)
return
@ -207,12 +205,7 @@ func (h *handler) DeleteBucketTaggingHandler(w http.ResponseWriter, r *http.Requ
w.WriteHeader(http.StatusNoContent)
}
func (h *handler) readTagSet(reader io.Reader) (map[string]string, error) {
tagging := new(Tagging)
if err := h.cfg.NewXMLDecoder(reader).Decode(tagging); err != nil {
return nil, errors.GetAPIError(errors.ErrMalformedXML)
}
func (h *handler) readTagSet(tagging *data.Tagging) (map[string]string, error) {
if err := checkTagSet(tagging.TagSet); err != nil {
return nil, err
}
@ -228,10 +221,10 @@ func (h *handler) readTagSet(reader io.Reader) (map[string]string, error) {
return tagSet, nil
}
func encodeTagging(tagSet map[string]string) *Tagging {
tagging := &Tagging{}
func encodeTagging(tagSet map[string]string) *data.Tagging {
tagging := &data.Tagging{}
for k, v := range tagSet {
tagging.TagSet = append(tagging.TagSet, Tag{Key: k, Value: v})
tagging.TagSet = append(tagging.TagSet, data.Tag{Key: k, Value: v})
}
sort.Slice(tagging.TagSet, func(i, j int) bool {
return tagging.TagSet[i].Key < tagging.TagSet[j].Key
@ -240,7 +233,7 @@ func encodeTagging(tagSet map[string]string) *Tagging {
return tagging
}
func checkTagSet(tagSet []Tag) error {
func checkTagSet(tagSet []data.Tag) error {
if len(tagSet) > maxTags {
return errors.GetAPIError(errors.ErrInvalidTagsSizeExceed)
}
@ -254,7 +247,7 @@ func checkTagSet(tagSet []Tag) error {
return nil
}
func checkTag(tag Tag) error {
func checkTag(tag data.Tag) error {
if len(tag.Key) < 1 || len(tag.Key) > keyTagMaxLength {
return errors.GetAPIError(errors.ErrInvalidTagKey)
}

41
api/handler/tagging_test.go
View file

@ -5,7 +5,9 @@ import (
"strings"
"testing"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
apiErrors "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/middleware"
"github.com/stretchr/testify/require"
)
@ -20,23 +22,23 @@ func TestTagsValidity(t *testing.T) {
}
for _, tc := range []struct {
tag Tag
tag data.Tag
valid bool
}{
{tag: Tag{}, valid: false},
{tag: Tag{Key: "", Value: "1"}, valid: false},
{tag: Tag{Key: "aws:key", Value: "val"}, valid: false},
{tag: Tag{Key: "key~", Value: "val"}, valid: false},
{tag: Tag{Key: "key\\", Value: "val"}, valid: false},
{tag: Tag{Key: "key?", Value: "val"}, valid: false},
{tag: Tag{Key: sbKey.String() + "b", Value: "val"}, valid: false},
{tag: Tag{Key: "key", Value: sbValue.String() + "b"}, valid: false},
{tag: data.Tag{}, valid: false},
{tag: data.Tag{Key: "", Value: "1"}, valid: false},
{tag: data.Tag{Key: "aws:key", Value: "val"}, valid: false},
{tag: data.Tag{Key: "key~", Value: "val"}, valid: false},
{tag: data.Tag{Key: "key\\", Value: "val"}, valid: false},
{tag: data.Tag{Key: "key?", Value: "val"}, valid: false},
{tag: data.Tag{Key: sbKey.String() + "b", Value: "val"}, valid: false},
{tag: data.Tag{Key: "key", Value: sbValue.String() + "b"}, valid: false},
{tag: Tag{Key: sbKey.String(), Value: "val"}, valid: true},
{tag: Tag{Key: "key", Value: sbValue.String()}, valid: true},
{tag: Tag{Key: "k e y", Value: "v a l"}, valid: true},
{tag: Tag{Key: "12345", Value: "1234"}, valid: true},
{tag: Tag{Key: allowedTagChars, Value: allowedTagChars}, valid: true},
{tag: data.Tag{Key: sbKey.String(), Value: "val"}, valid: true},
{tag: data.Tag{Key: "key", Value: sbValue.String()}, valid: true},
{tag: data.Tag{Key: "k e y", Value: "v a l"}, valid: true},
{tag: data.Tag{Key: "12345", Value: "1234"}, valid: true},
{tag: data.Tag{Key: allowedTagChars, Value: allowedTagChars}, valid: true},
} {
err := checkTag(tc.tag)
if tc.valid {
@ -55,13 +57,13 @@ func TestPutObjectTaggingCheckUniqueness(t *testing.T) {
for _, tc := range []struct {
name string
body *Tagging
body *data.Tagging
error bool
}{
{
name: "Two tags with unique keys",
body: &Tagging{
TagSet: []Tag{
body: &data.Tagging{
TagSet: []data.Tag{
{
Key: "key-1",
Value: "val-1",
@ -76,8 +78,8 @@ func TestPutObjectTaggingCheckUniqueness(t *testing.T) {
},
{
name: "Two tags with the same keys",
body: &Tagging{
TagSet: []Tag{
body: &data.Tagging{
TagSet: []data.Tag{
{
Key: "key-1",
Value: "val-1",
@ -93,6 +95,7 @@ func TestPutObjectTaggingCheckUniqueness(t *testing.T) {
} {
t.Run(tc.name, func(t *testing.T) {
w, r := prepareTestRequest(hc, bktName, objName, tc.body)
middleware.GetReqInfo(r.Context()).Tagging = tc.body
hc.Handler().PutObjectTaggingHandler(w, r)
if tc.error {
assertS3Error(t, w, apiErrors.GetAPIError(apiErrors.ErrInvalidTagKeyUniqueness))

2
api/layer/compound.go
View file

@ -9,7 +9,7 @@ import (
s3errors "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
)
func (n *layer) GetObjectTaggingAndLock(ctx context.Context, objVersion *ObjectVersion, nodeVersion *data.NodeVersion) (map[string]string, data.LockInfo, error) {
func (n *layer) GetObjectTaggingAndLock(ctx context.Context, objVersion *data.ObjectVersion, nodeVersion *data.NodeVersion) (map[string]string, data.LockInfo, error) {
var err error
owner := n.BearerOwner(ctx)

22
api/layer/layer.go
View file

@ -97,14 +97,6 @@ type (
VersionID string
}
// ObjectVersion stores object version info.
ObjectVersion struct {
BktInfo *data.BucketInfo
ObjectName string
VersionID string
NoErrorOnDeleteMarker bool
}
// RangeParams stores range header request parameters.
RangeParams struct {
Start uint64
@ -244,16 +236,16 @@ type (
GetObjectInfo(ctx context.Context, p *HeadObjectParams) (*data.ObjectInfo, error)
GetExtendedObjectInfo(ctx context.Context, p *HeadObjectParams) (*data.ExtendedObjectInfo, error)
GetLockInfo(ctx context.Context, obj *ObjectVersion) (*data.LockInfo, error)
GetLockInfo(ctx context.Context, obj *data.ObjectVersion) (*data.LockInfo, error)
PutLockInfo(ctx context.Context, p *PutLockInfoParams) error
GetBucketTagging(ctx context.Context, bktInfo *data.BucketInfo) (map[string]string, error)
PutBucketTagging(ctx context.Context, bktInfo *data.BucketInfo, tagSet map[string]string) error
DeleteBucketTagging(ctx context.Context, bktInfo *data.BucketInfo) error
GetObjectTagging(ctx context.Context, p *GetObjectTaggingParams) (string, map[string]string, error)
PutObjectTagging(ctx context.Context, p *PutObjectTaggingParams) (*data.NodeVersion, error)
DeleteObjectTagging(ctx context.Context, p *ObjectVersion) (*data.NodeVersion, error)
GetObjectTagging(ctx context.Context, p *data.GetObjectTaggingParams) (string, map[string]string, error)
PutObjectTagging(ctx context.Context, p *data.PutObjectTaggingParams) (*data.NodeVersion, error)
DeleteObjectTagging(ctx context.Context, p *data.ObjectVersion) (*data.NodeVersion, error)
PutObject(ctx context.Context, p *PutObjectParams) (*data.ExtendedObjectInfo, error)
@ -279,7 +271,7 @@ type (
// Compound methods for optimizations
// GetObjectTaggingAndLock unifies GetObjectTagging and GetLock methods in single tree service invocation.
GetObjectTaggingAndLock(ctx context.Context, p *ObjectVersion, nodeVersion *data.NodeVersion) (map[string]string, data.LockInfo, error)
GetObjectTaggingAndLock(ctx context.Context, p *data.ObjectVersion, nodeVersion *data.NodeVersion) (map[string]string, data.LockInfo, error)
}
)
@ -765,7 +757,7 @@ func isNotFoundError(err error) bool {
}
func (n *layer) getNodeVersionToDelete(ctx context.Context, bkt *data.BucketInfo, obj *VersionedObject) (*data.NodeVersion, error) {
objVersion := &ObjectVersion{
objVersion := &data.ObjectVersion{
BktInfo: bkt,
ObjectName: obj.Name,
VersionID: obj.VersionID,
@ -776,7 +768,7 @@ func (n *layer) getNodeVersionToDelete(ctx context.Context, bkt *data.BucketInfo
}
func (n *layer) getLastNodeVersion(ctx context.Context, bkt *data.BucketInfo, obj *VersionedObject) (*data.NodeVersion, error) {
objVersion := &ObjectVersion{
objVersion := &data.ObjectVersion{
BktInfo: bkt,
ObjectName: obj.Name,
VersionID: "",

2
api/layer/locking_test.go
View file

@ -20,7 +20,7 @@ func TestObjectLockAttributes(t *testing.T) {
obj := tc.putObject([]byte("content obj1 v1"))
p := &PutLockInfoParams{
ObjVersion: &ObjectVersion{
ObjVersion: &data.ObjectVersion{
BktInfo: tc.bktInfo,
ObjectName: obj.Name,
VersionID: obj.VersionID(),

2
api/layer/object.go
View file

@ -321,7 +321,7 @@ func (n *layer) PutObject(ctx context.Context, p *PutObjectParams) (*data.Extend
if p.Lock != nil && (p.Lock.Retention != nil || p.Lock.LegalHold != nil) {
putLockInfoPrms := &PutLockInfoParams{
ObjVersion: &ObjectVersion{
ObjVersion: &data.ObjectVersion{
BktInfo: p.BktInfo,
ObjectName: p.Object,
VersionID: id.EncodeToString(),

8
api/layer/system_object.go
View file

@ -20,7 +20,7 @@ const (
)
type PutLockInfoParams struct {
ObjVersion *ObjectVersion
ObjVersion *data.ObjectVersion
NewLock *data.ObjectLock
CopiesNumbers []uint32
NodeVersion *data.NodeVersion // optional
@ -100,7 +100,7 @@ func (n *layer) PutLockInfo(ctx context.Context, p *PutLockInfoParams) (err erro
return nil
}
func (n *layer) getNodeVersionFromCacheOrFrostfs(ctx context.Context, objVersion *ObjectVersion) (nodeVersion *data.NodeVersion, err error) {
func (n *layer) getNodeVersionFromCacheOrFrostfs(ctx context.Context, objVersion *data.ObjectVersion) (nodeVersion *data.NodeVersion, err error) {
// check cache if node version is stored inside extendedObjectVersion
nodeVersion = n.getNodeVersionFromCache(n.BearerOwner(ctx), objVersion)
if nodeVersion == nil {
@ -129,7 +129,7 @@ func (n *layer) putLockObject(ctx context.Context, bktInfo *data.BucketInfo, obj
return id, err
}
func (n *layer) GetLockInfo(ctx context.Context, objVersion *ObjectVersion) (*data.LockInfo, error) {
func (n *layer) GetLockInfo(ctx context.Context, objVersion *data.ObjectVersion) (*data.LockInfo, error) {
owner := n.BearerOwner(ctx)
if lockInfo := n.cache.GetLockInfo(owner, lockObjectKey(objVersion)); lockInfo != nil {
return lockInfo, nil
@ -185,7 +185,7 @@ func (n *layer) getCORS(ctx context.Context, bkt *data.BucketInfo) (*data.CORSCo
return cors, nil
}
func lockObjectKey(objVersion *ObjectVersion) string {
func lockObjectKey(objVersion *data.ObjectVersion) string {
// todo reconsider forming name since versionID can be "null" or ""
return ".lock." + objVersion.BktInfo.CID.EncodeToString() + "." + objVersion.ObjectName + "." + objVersion.VersionID
}

27
api/layer/tagging.go
View file

@ -14,22 +14,7 @@ import (
"go.uber.org/zap"
)
type GetObjectTaggingParams struct {
ObjectVersion *ObjectVersion
// NodeVersion can be nil. If not nil we save one request to tree service.
NodeVersion *data.NodeVersion // optional
}
type PutObjectTaggingParams struct {
ObjectVersion *ObjectVersion
TagSet map[string]string
// NodeVersion can be nil. If not nil we save one request to tree service.
NodeVersion *data.NodeVersion // optional
}
func (n *layer) GetObjectTagging(ctx context.Context, p *GetObjectTaggingParams) (string, map[string]string, error) {
func (n *layer) GetObjectTagging(ctx context.Context, p *data.GetObjectTaggingParams) (string, map[string]string, error) {
var err error
owner := n.BearerOwner(ctx)
@ -65,7 +50,7 @@ func (n *layer) GetObjectTagging(ctx context.Context, p *GetObjectTaggingParams)
return p.ObjectVersion.VersionID, tags, nil
}
func (n *layer) PutObjectTagging(ctx context.Context, p *PutObjectTaggingParams) (nodeVersion *data.NodeVersion, err error) {
func (n *layer) PutObjectTagging(ctx context.Context, p *data.PutObjectTaggingParams) (nodeVersion *data.NodeVersion, err error) {
nodeVersion = p.NodeVersion
if nodeVersion == nil {
nodeVersion, err = n.getNodeVersionFromCacheOrFrostfs(ctx, p.ObjectVersion)
@ -88,7 +73,7 @@ func (n *layer) PutObjectTagging(ctx context.Context, p *PutObjectTaggingParams)
return nodeVersion, nil
}
func (n *layer) DeleteObjectTagging(ctx context.Context, p *ObjectVersion) (*data.NodeVersion, error) {
func (n *layer) DeleteObjectTagging(ctx context.Context, p *data.ObjectVersion) (*data.NodeVersion, error) {
version, err := n.getNodeVersion(ctx, p)
if err != nil {
return nil, err
@ -142,7 +127,7 @@ func (n *layer) DeleteBucketTagging(ctx context.Context, bktInfo *data.BucketInf
return n.treeService.DeleteBucketTagging(ctx, bktInfo)
}
func objectTaggingCacheKey(p *ObjectVersion) string {
func objectTaggingCacheKey(p *data.ObjectVersion) string {
return ".tagset." + p.BktInfo.CID.EncodeToString() + "." + p.ObjectName + "." + p.VersionID
}
@ -150,7 +135,7 @@ func bucketTaggingCacheKey(cnrID cid.ID) string {
return ".tagset." + cnrID.EncodeToString()
}
func (n *layer) getNodeVersion(ctx context.Context, objVersion *ObjectVersion) (*data.NodeVersion, error) {
func (n *layer) getNodeVersion(ctx context.Context, objVersion *data.ObjectVersion) (*data.NodeVersion, error) {
var err error
var version *data.NodeVersion
@ -188,7 +173,7 @@ func (n *layer) getNodeVersion(ctx context.Context, objVersion *ObjectVersion) (
return version, err
}
func (n *layer) getNodeVersionFromCache(owner user.ID, o *ObjectVersion) *data.NodeVersion {
func (n *layer) getNodeVersionFromCache(owner user.ID, o *data.ObjectVersion) *data.NodeVersion {
if len(o.VersionID) == 0 || o.VersionID == data.UnversionedObjectVersionID {
return nil
}

152
api/middleware/policy.go
View file

@ -3,12 +3,16 @@ package middleware
import (
"context"
"crypto/elliptic"
"encoding/xml"
"fmt"
"io"
"net/http"
"net/url"
"strings"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
apiErr "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
frostfsErrors "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/frostfs/errors"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs"
"git.frostfs.info/TrueCloudLab/policy-engine/pkg/chain"
"git.frostfs.info/TrueCloudLab/policy-engine/pkg/engine"
@ -25,8 +29,22 @@ const (
QueryPrefix = "prefix"
QueryDelimiter = "delimiter"
QueryMaxKeys = "max-keys"
amzTagging = "x-amz-tagging"
)
// At the beginning of these operations resources haven't yet been created.
var withoutResourceOps = []string{
CreateBucketOperation,
CreateMultipartUploadOperation,
AbortMultipartUploadOperation,
CompleteMultipartUploadOperation,
UploadPartOperation,
UploadPartCopyOperation,
ListPartsOperation,
dkirillov marked this conversation as resolved Outdated
dkirillov commented 2024-04-11 06:39:42 +00:00
Outdated
Review
Copy link

Can aws restrict uploadPart/listingParts by resource attribute (that was added during create multipart upload)?

Can aws restrict uploadPart/listingParts by resource attribute (that was added during create multipart upload)?
mbiryukova commented 2024-04-11 10:43:08 +00:00
Outdated
Review
Copy link

It seems not

It seems not
👍 1
PutObjectOperation,
CopyObjectOperation,
}
type PolicySettings interface {
PolicyDenyByDefault() bool
ACLEnabled() bool
@ -36,6 +54,15 @@ type FrostFSIDInformer interface {
GetUserGroupIDs(userHash util.Uint160) ([]string, error)
}
type XMLDecoder interface {
NewXMLDecoder(io.Reader) *xml.Decoder
}
type ResourceTagging interface {
GetBucketTagging(ctx context.Context, bktInfo *data.BucketInfo) (map[string]string, error)
dkirillov marked this conversation as resolved Outdated
dkirillov commented 2024-04-11 06:43:50 +00:00
Outdated
Review
Copy link

This duplicates PolicyConfig.BucketResolver.

This duplicates `PolicyConfig.BucketResolver`.
GetObjectTagging(ctx context.Context, p *data.GetObjectTaggingParams) (string, map[string]string, error)
}
// BucketResolveFunc is a func to resolve bucket info by name.
type BucketResolveFunc func(ctx context.Context, bucket string) (*data.BucketInfo, error)
@ -46,6 +73,8 @@ type PolicyConfig struct {
Domains []string
Log *zap.Logger
BucketResolver BucketResolveFunc
Decoder XMLDecoder
Tagging ResourceTagging
}
func PolicyCheck(cfg PolicyConfig) Func {
@ -54,6 +83,7 @@ func PolicyCheck(cfg PolicyConfig) Func {
ctx := r.Context()
if err := policyCheck(r, cfg); err != nil {
reqLogOrDefault(ctx, cfg.Log).Error(logs.PolicyValidationFailed, zap.Error(err))
err = frostfsErrors.UnwrapErr(err)
if _, wrErr := WriteErrorResponse(w, GetReqInfo(ctx), err); wrErr != nil {
dkirillov marked this conversation as resolved Outdated
dkirillov commented 2024-04-11 06:38:42 +00:00
Outdated
Review
Copy link

Why do we need this?

Why do we need this?
mbiryukova commented 2024-04-11 08:41:00 +00:00
Outdated
Review
Copy link

Without this we will return InternalError if requesting tags of non-existent bucket or object, instead of NoSuchBucket or NoSuchKey

Without this we will return `InternalError` if requesting tags of non-existent bucket or object, instead of `NoSuchBucket` or `NoSuchKey`
dkirillov commented 2024-04-11 09:13:59 +00:00
Outdated
Review
Copy link

Can we add test for such case then? For now If I remove this line tests still pass

Can we add test for such case then? For now If I remove this line tests still pass
reqLogOrDefault(ctx, cfg.Log).Error(logs.FailedToWriteResponse, zap.Error(wrErr))
}
@ -67,7 +97,7 @@ func PolicyCheck(cfg PolicyConfig) Func {
func policyCheck(r *http.Request, cfg PolicyConfig) error {
reqType, bktName, objName := getBucketObject(r, cfg.Domains)
req, err := getPolicyRequest(r, cfg.FrostfsID, reqType, bktName, objName, cfg.Log)
req, err := getPolicyRequest(r, cfg, reqType, bktName, objName)
if err != nil {
dkirillov marked this conversation as resolved Outdated
dkirillov commented 2024-04-11 06:45:22 +00:00
Outdated
Review
Copy link

It seems we can pass whole cfg param

It seems we can pass whole `cfg` param
return err
}
@ -115,7 +145,7 @@ func policyCheck(r *http.Request, cfg PolicyConfig) error {
return nil
}
func getPolicyRequest(r *http.Request, frostfsid FrostFSIDInformer, reqType ReqType, bktName string, objName string, log *zap.Logger) (*testutil.Request, error) {
func getPolicyRequest(r *http.Request, cfg PolicyConfig, reqType ReqType, bktName string, objName string) (*testutil.Request, error) {
var (
owner string
groups []string
@ -130,7 +160,7 @@ func getPolicyRequest(r *http.Request, frostfsid FrostFSIDInformer, reqType ReqT
}
owner = pk.Address()
groups, err = frostfsid.GetUserGroupIDs(pk.GetScriptHash())
groups, err = cfg.FrostfsID.GetUserGroupIDs(pk.GetScriptHash())
if err != nil {
return nil, fmt.Errorf("get group ids: %w", err)
}
@ -145,9 +175,12 @@ func getPolicyRequest(r *http.Request, frostfsid FrostFSIDInformer, reqType ReqT
res = fmt.Sprintf(s3.ResourceFormatS3Bucket, bktName)
}
properties := determineProperties(ctx, reqType, op, owner, groups)
properties, err := determineProperties(r, cfg.Decoder, cfg.BucketResolver, cfg.Tagging, reqType, op, bktName, objName, owner, groups)
if err != nil {
return nil, fmt.Errorf("determine properties: %w", err)
}
reqLogOrDefault(r.Context(), log).Debug(logs.PolicyRequest, zap.String("action", op),
reqLogOrDefault(r.Context(), cfg.Log).Debug(logs.PolicyRequest, zap.String("action", op),
zap.String("resource", res), zap.Any("properties", properties))
return testutil.NewRequest(op, testutil.NewResource(res, nil), properties), nil
@ -376,12 +409,13 @@ func determineGeneralOperation(r *http.Request) string {
return "UnmatchedOperation"
}
func determineProperties(ctx context.Context, reqType ReqType, op, owner string, groups []string) map[string]string {
func determineProperties(r *http.Request, decoder XMLDecoder, resolver BucketResolveFunc, tagging ResourceTagging, reqType ReqType,
op, bktName, objName, owner string, groups []string) (map[string]string, error) {
res := map[string]string{
s3.PropertyKeyOwner: owner,
common.PropertyKeyFrostFSIDGroupID: chain.FormCondSliceContainsValue(groups),
}
queries := GetReqInfo(ctx).URL.Query()
queries := GetReqInfo(r.Context()).URL.Query()
if reqType == objectType {
if versionID := queries.Get(QueryVersionID); len(versionID) > 0 {
@ -402,5 +436,107 @@ func determineProperties(ctx context.Context, reqType ReqType, op, owner string,
}
}
return res
tags, err := determineTags(r, decoder, resolver, tagging, reqType, op, bktName, objName, queries.Get(QueryVersionID))
if err != nil {
return nil, fmt.Errorf("determine tags: %w", err)
}
for k, v := range tags {
res[k] = v
}
return res, nil
}
func determineTags(r *http.Request, decoder XMLDecoder, resolver BucketResolveFunc, tagging ResourceTagging, reqType ReqType,
op, bktName, objName, versionID string) (map[string]string, error) {
res, err := determineRequestTags(r, decoder, op)
if err != nil {
return nil, fmt.Errorf("determine request tags: %w", err)
}
tags, err := determineResourceTags(r.Context(), reqType, op, bktName, objName, versionID, resolver, tagging)
if err != nil {
return nil, fmt.Errorf("determine resource tags: %w", err)
}
for k, v := range tags {
res[k] = v
}
return res, nil
}
func determineRequestTags(r *http.Request, decoder XMLDecoder, op string) (map[string]string, error) {
tags := make(map[string]string)
if strings.HasSuffix(op, PutObjectTaggingOperation) || strings.HasSuffix(op, PutBucketTaggingOperation) {
tagging := new(data.Tagging)
if err := decoder.NewXMLDecoder(r.Body).Decode(tagging); err != nil {
return nil, apiErr.GetAPIError(apiErr.ErrMalformedXML)
}
GetReqInfo(r.Context()).Tagging = tagging
for _, tag := range tagging.TagSet {
tags[fmt.Sprintf(s3.PropertyKeyFormatRequestTag, tag.Key)] = tag.Value
}
}
if tagging := r.Header.Get(amzTagging); len(tagging) > 0 {
queries, err := url.ParseQuery(tagging)
if err != nil {
return nil, apiErr.GetAPIError(apiErr.ErrInvalidArgument)
}
for key := range queries {
tags[fmt.Sprintf(s3.PropertyKeyFormatRequestTag, key)] = queries.Get(key)
}
}
return tags, nil
}
func determineResourceTags(ctx context.Context, reqType ReqType, op, bktName, objName, versionID string, resolver BucketResolveFunc,
tagging ResourceTagging) (map[string]string, error) {
tags := make(map[string]string)
if reqType != bucketType && reqType != objectType {
return tags, nil
}
for _, withoutResOp := range withoutResourceOps {
if strings.HasSuffix(op, withoutResOp) {
return tags, nil
}
}
bktInfo, err := resolver(ctx, bktName)
if err != nil {
return nil, fmt.Errorf("get bucket info: %w", err)
}
if reqType == bucketType {
tags, err = tagging.GetBucketTagging(ctx, bktInfo)
if err != nil {
return nil, fmt.Errorf("get bucket tagging: %w", err)
}
}
if reqType == objectType {
tagPrm := &data.GetObjectTaggingParams{
ObjectVersion: &data.ObjectVersion{
BktInfo: bktInfo,
ObjectName: objName,
VersionID: versionID,
},
}
_, tags, err = tagging.GetObjectTagging(ctx, tagPrm)
if err != nil {
return nil, fmt.Errorf("get object tagging: %w", err)
}
}
res := make(map[string]string, len(tags))
for k, v := range tags {
res[fmt.Sprintf(s3.PropertyKeyFormatResourceTag, k)] = v
}
return res, nil
}

1
api/middleware/reqinfo.go
View file

@ -40,6 +40,7 @@ type (
URL *url.URL // Request url
Namespace string
User string // User owner id
Tagging *data.Tagging
tags []KeyVal // Any additional info not accommodated by above fields
}

5
api/router.go
View file

@ -121,6 +121,9 @@ type Config struct {
FrostFSIDValidation bool
PolicyChecker engine.ChainRouter
XMLDecoder s3middleware.XMLDecoder
Tagging s3middleware.ResourceTagging
}
func NewRouter(cfg Config) *chi.Mux {
@ -146,6 +149,8 @@ func NewRouter(cfg Config) *chi.Mux {
Domains: cfg.Domains,
Log: cfg.Log,
BucketResolver: cfg.Handler.ResolveBucket,
Decoder: cfg.XMLDecoder,
Tagging: cfg.Tagging,
}))
defaultRouter := chi.NewRouter()

50
api/router_mock_test.go
View file

@ -3,11 +3,13 @@ package api
import (
"context"
"encoding/json"
"errors"
"encoding/xml"
"io"
"net/http"
"testing"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
apiErrors "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/middleware"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/creds/accessbox"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer"
@ -86,6 +88,30 @@ func (f *frostFSIDMock) GetUserGroupIDs(util.Uint160) ([]string, error) {
return []string{}, nil
}
type xmlMock struct {
}
func (m *xmlMock) NewXMLDecoder(r io.Reader) *xml.Decoder {
return xml.NewDecoder(r)
}
type resourceTaggingMock struct {
bucketTags map[string]string
objectTags map[string]string
noSuchKey bool
}
func (m *resourceTaggingMock) GetBucketTagging(context.Context, *data.BucketInfo) (map[string]string, error) {
return m.bucketTags, nil
}
func (m *resourceTaggingMock) GetObjectTagging(context.Context, *data.GetObjectTaggingParams) (string, map[string]string, error) {
if m.noSuchKey {
return "", nil, apiErrors.GetAPIError(apiErrors.ErrNoSuchKey)
}
return "", m.objectTags, nil
}
type handlerMock struct {
t *testing.T
cfg *middlewareSettingsMock
@ -142,9 +168,13 @@ func (h *handlerMock) GetObjectLegalHoldHandler(http.ResponseWriter, *http.Reque
panic("implement me")
}
func (h *handlerMock) GetObjectHandler(http.ResponseWriter, *http.Request) {
//TODO implement me
panic("implement me")
func (h *handlerMock) GetObjectHandler(w http.ResponseWriter, r *http.Request) {
res := &handlerResult{
Method: middleware.GetObjectOperation,
ReqInfo: middleware.GetReqInfo(r.Context()),
}
h.writeResponse(w, res)
}
func (h *handlerMock) GetObjectAttributesHandler(http.ResponseWriter, *http.Request) {
@ -339,9 +369,13 @@ func (h *handlerMock) PutBucketObjectLockConfigHandler(http.ResponseWriter, *htt
panic("implement me")
}
func (h *handlerMock) PutBucketTaggingHandler(http.ResponseWriter, *http.Request) {
//TODO implement me
panic("implement me")
func (h *handlerMock) PutBucketTaggingHandler(w http.ResponseWriter, r *http.Request) {
res := &handlerResult{
Method: middleware.PutBucketTaggingOperation,
ReqInfo: middleware.GetReqInfo(r.Context()),
}
h.writeResponse(w, res)
}
func (h *handlerMock) PutBucketVersioningHandler(http.ResponseWriter, *http.Request) {
@ -473,7 +507,7 @@ func (h *handlerMock) ResolveBucket(ctx context.Context, name string) (*data.Buc
reqInfo := middleware.GetReqInfo(ctx)
bktInfo, ok := h.buckets[reqInfo.Namespace+name]
if !ok {
return nil, errors.New("not found")
return nil, apiErrors.GetAPIError(apiErrors.ErrNoSuchBucket)
}
return bktInfo, nil
}

207
api/router_test.go
View file

@ -1,6 +1,7 @@
package api
import (
"bytes"
"encoding/json"
"encoding/xml"
"fmt"
@ -66,6 +67,8 @@ func prepareRouter(t *testing.T) *routerMock {
PolicyChecker: policyChecker,
Domains: []string{"domain1", "domain2"},
FrostfsID: &frostFSIDMock{},
XMLDecoder: &xmlMock{},
Tagging: &resourceTaggingMock{},
}
return &routerMock{
t: t,
@ -115,7 +118,7 @@ func TestRouterObjectWithSlashes(t *testing.T) {
ns, bktName, objName := "", "dkirillov", "/fix/object"
createBucket(chiRouter, ns, bktName)
resp := putObject(chiRouter, ns, bktName, objName)
resp := putObject(chiRouter, ns, bktName, objName, nil)
require.Equal(t, objName, resp.ReqInfo.ObjectName)
}
@ -157,7 +160,7 @@ func TestRouterObjectEscaping(t *testing.T) {
},
} {
t.Run(tc.name, func(t *testing.T) {
resp := putObject(chiRouter, ns, bktName, tc.objName)
resp := putObject(chiRouter, ns, bktName, tc.objName, nil)
require.Equal(t, tc.expectedObjName, resp.ReqInfo.ObjectName)
})
}
@ -185,13 +188,13 @@ func TestPolicyChecker(t *testing.T) {
require.NoError(t, err)
// check we can access 'bucket' in default namespace
putObject(chiRouter, ns1, bktName1, objName1)
putObject(chiRouter, ns1, bktName1, objName1, nil)
// check we can access 'other-bucket' in custom namespace
putObject(chiRouter, ns2, bktName2, objName2)
putObject(chiRouter, ns2, bktName2, objName2, nil)
// check we cannot access 'bucket' in custom namespace
putObjectErr(chiRouter, ns2, bktName1, objName2, apiErrors.ErrAccessDenied)
putObjectErr(chiRouter, ns2, bktName1, objName2, nil, apiErrors.ErrAccessDenied)
}
func TestPolicyCheckerReqTypeDetermination(t *testing.T) {
@ -215,6 +218,8 @@ func TestPolicyCheckerReqTypeDetermination(t *testing.T) {
_, _, err = chiRouter.policyChecker.MorphRuleChainStorage().AddMorphRuleChain(chain.S3, engine.NamespaceTarget(""), ruleChain)
require.NoError(t, err)
createBucket(chiRouter, "", bktName)
chiRouter.middlewareSettings.denyByDefault = true
t.Run("can list buckets", func(t *testing.T) {
w, r := httptest.NewRecorder(), httptest.NewRequest(http.MethodGet, "/", nil)
@ -263,9 +268,9 @@ func TestACLAPE(t *testing.T) {
router.middlewareSettings.denyByDefault = true
// Allow because of using old bucket
putObject(router, ns, bktNameOld, objName)
putObject(router, ns, bktNameOld, objName, nil)
// Deny because of deny by default
putObjectErr(router, ns, bktNameNew, objName, apiErrors.ErrAccessDenied)
putObjectErr(router, ns, bktNameNew, objName, nil, apiErrors.ErrAccessDenied)
// Deny because of deny by default
createBucketErr(router, ns, bktName, apiErrors.ErrAccessDenied)
@ -289,9 +294,9 @@ func TestACLAPE(t *testing.T) {
router.middlewareSettings.denyByDefault = false
// Allow because of using old bucket
putObject(router, ns, bktNameOld, objName)
putObject(router, ns, bktNameOld, objName, nil)
// Allow because of allow by default
putObject(router, ns, bktNameNew, objName)
putObject(router, ns, bktNameNew, objName, nil)
// Allow because of deny by default
createBucket(router, ns, bktName)
@ -315,9 +320,9 @@ func TestACLAPE(t *testing.T) {
router.middlewareSettings.denyByDefault = true
// Allow because of using old bucket
putObject(router, ns, bktNameOld, objName)
putObject(router, ns, bktNameOld, objName, nil)
// Deny because of deny by default
putObjectErr(router, ns, bktNameNew, objName, apiErrors.ErrAccessDenied)
putObjectErr(router, ns, bktNameNew, objName, nil, apiErrors.ErrAccessDenied)
// Allow because of old behavior
createBucket(router, ns, bktName)
@ -336,9 +341,9 @@ func TestACLAPE(t *testing.T) {
router.middlewareSettings.denyByDefault = false
// Allow because of using old bucket
putObject(router, ns, bktNameOld, objName)
putObject(router, ns, bktNameOld, objName, nil)
// Allow because of allow by default
putObject(router, ns, bktNameNew, objName)
putObject(router, ns, bktNameNew, objName, nil)
// Allow because of old behavior
createBucket(router, ns, bktName)
@ -459,6 +464,118 @@ func TestRequestParametersCheck(t *testing.T) {
})
}
func TestRequestTagsCheck(t *testing.T) {
t.Run("put bucket tagging", func(t *testing.T) {
router := prepareRouter(t)
ns, bktName, tagKey, tagValue := "", "bucket", "tag", "value"
router.middlewareSettings.denyByDefault = true
allowOperations(router, ns, []string{"s3:CreateBucket"}, nil)
createBucket(router, ns, bktName)
// Add policies and check
allowOperations(router, ns, []string{"s3:PutBucketTagging"}, engineiam.Conditions{
engineiam.CondStringEquals: engineiam.Condition{fmt.Sprintf(s3.PropertyKeyFormatRequestTag, tagKey): []string{tagValue}},
})
denyOperations(router, ns, []string{"s3:PutBucketTagging"}, engineiam.Conditions{
engineiam.CondStringNotEquals: engineiam.Condition{fmt.Sprintf(s3.PropertyKeyFormatRequestTag, tagKey): []string{tagValue}},
})
tagging, err := xml.Marshal(data.Tagging{TagSet: []data.Tag{{Key: tagKey, Value: tagValue}}})
require.NoError(t, err)
putBucketTagging(router, ns, bktName, tagging)
tagging, err = xml.Marshal(data.Tagging{TagSet: []data.Tag{{Key: "key", Value: tagValue}}})
require.NoError(t, err)
putBucketTaggingErr(router, ns, bktName, tagging, apiErrors.ErrAccessDenied)
})
t.Run("put object with tag", func(t *testing.T) {
router := prepareRouter(t)
ns, bktName, objName, tagKey, tagValue := "", "bucket", "object", "tag", "value"
router.middlewareSettings.denyByDefault = true
allowOperations(router, ns, []string{"s3:CreateBucket"}, nil)
createBucket(router, ns, bktName)
// Add policies and check
allowOperations(router, ns, []string{"s3:PutObject"}, engineiam.Conditions{
engineiam.CondStringEquals: engineiam.Condition{fmt.Sprintf(s3.PropertyKeyFormatRequestTag, tagKey): []string{tagValue}},
})
denyOperations(router, ns, []string{"s3:PutObject"}, engineiam.Conditions{
engineiam.CondStringNotEquals: engineiam.Condition{fmt.Sprintf(s3.PropertyKeyFormatRequestTag, tagKey): []string{tagValue}},
})
putObject(router, ns, bktName, objName, &data.Tag{Key: tagKey, Value: tagValue})
putObjectErr(router, ns, bktName, objName, &data.Tag{Key: "key", Value: tagValue}, apiErrors.ErrAccessDenied)
})
}
func TestResourceTagsCheck(t *testing.T) {
t.Run("bucket tagging", func(t *testing.T) {
router := prepareRouter(t)
ns, bktName, tagKey, tagValue := "", "bucket", "tag", "value"
router.middlewareSettings.denyByDefault = true
allowOperations(router, ns, []string{"s3:CreateBucket"}, nil)
createBucket(router, ns, bktName)
// Add policies and check
allowOperations(router, ns, []string{"s3:ListBucket"}, engineiam.Conditions{
engineiam.CondStringEquals: engineiam.Condition{fmt.Sprintf(s3.PropertyKeyFormatResourceTag, tagKey): []string{tagValue}},
})
denyOperations(router, ns, []string{"s3:ListBucket"}, engineiam.Conditions{
engineiam.CondStringNotEquals: engineiam.Condition{fmt.Sprintf(s3.PropertyKeyFormatResourceTag, tagKey): []string{tagValue}},
})
router.cfg.Tagging.(*resourceTaggingMock).bucketTags = map[string]string{tagKey: tagValue}
listObjectsV1(router, ns, bktName, "", "", "")
router.cfg.Tagging.(*resourceTaggingMock).bucketTags = map[string]string{}
listObjectsV1Err(router, ns, bktName, "", "", "", apiErrors.ErrAccessDenied)
})
t.Run("object tagging", func(t *testing.T) {
router := prepareRouter(t)
ns, bktName, objName, tagKey, tagValue := "", "bucket", "object", "tag", "value"
router.middlewareSettings.denyByDefault = true
allowOperations(router, ns, []string{"s3:CreateBucket", "s3:PutObject"}, nil)
createBucket(router, ns, bktName)
putObject(router, ns, bktName, objName, nil)
// Add policies and check
allowOperations(router, ns, []string{"s3:GetObject"}, engineiam.Conditions{
engineiam.CondStringEquals: engineiam.Condition{fmt.Sprintf(s3.PropertyKeyFormatResourceTag, tagKey): []string{tagValue}},
})
denyOperations(router, ns, []string{"s3:GetObject"}, engineiam.Conditions{
engineiam.CondStringNotEquals: engineiam.Condition{fmt.Sprintf(s3.PropertyKeyFormatResourceTag, tagKey): []string{tagValue}},
})
router.cfg.Tagging.(*resourceTaggingMock).objectTags = map[string]string{tagKey: tagValue}
getObject(router, ns, bktName, objName)
router.cfg.Tagging.(*resourceTaggingMock).objectTags = map[string]string{}
getObjectErr(router, ns, bktName, objName, apiErrors.ErrAccessDenied)
})
t.Run("non-existent resources", func(t *testing.T) {
router := prepareRouter(t)
ns, bktName, objName := "", "bucket", "object"
listObjectsV1Err(router, ns, bktName, "", "", "", apiErrors.ErrNoSuchBucket)
router.cfg.Tagging.(*resourceTaggingMock).noSuchKey = true
createBucket(router, ns, bktName)
getObjectErr(router, ns, bktName, objName, apiErrors.ErrNoSuchKey)
})
}
func allowOperations(router *routerMock, ns string, operations []string, conditions engineiam.Conditions) {
addPolicy(router, ns, "allow", engineiam.AllowEffect, operations, conditions)
}
@ -538,20 +655,68 @@ func listBucketsBase(router *routerMock, namespace string) *httptest.ResponseRec
return w
}
func putObject(router *routerMock, namespace, bktName, objName string) handlerResult {
w := putObjectBase(router, namespace, bktName, objName)
func putObject(router *routerMock, namespace, bktName, objName string, tag *data.Tag) handlerResult {
w := putObjectBase(router, namespace, bktName, objName, tag)
resp := readResponse(router.t, w)
require.Equal(router.t, s3middleware.PutObjectOperation, resp.Method)
return resp
}
func putObjectErr(router *routerMock, namespace, bktName, objName string, errCode apiErrors.ErrorCode) {
w := putObjectBase(router, namespace, bktName, objName)
func putObjectErr(router *routerMock, namespace, bktName, objName string, tag *data.Tag, errCode apiErrors.ErrorCode) {
w := putObjectBase(router, namespace, bktName, objName, tag)
assertAPIError(router.t, w, errCode)
}
func putObjectBase(router *routerMock, namespace, bktName, objName string) *httptest.ResponseRecorder {
func putObjectBase(router *routerMock, namespace, bktName, objName string, tag *data.Tag) *httptest.ResponseRecorder {
w, r := httptest.NewRecorder(), httptest.NewRequest(http.MethodPut, "/"+bktName+"/"+objName, nil)
if tag != nil {
queries := url.Values{
tag.Key: []string{tag.Value},
}
r.Header.Set(AmzTagging, queries.Encode())
}
r.Header.Set(FrostfsNamespaceHeader, namespace)
router.ServeHTTP(w, r)
return w
}
func putBucketTagging(router *routerMock, namespace, bktName string, tagging []byte) handlerResult {
w := putBucketTaggingBase(router, namespace, bktName, tagging)
resp := readResponse(router.t, w)
require.Equal(router.t, s3middleware.PutBucketTaggingOperation, resp.Method)
return resp
}
func putBucketTaggingErr(router *routerMock, namespace, bktName string, tagging []byte, errCode apiErrors.ErrorCode) {
w := putBucketTaggingBase(router, namespace, bktName, tagging)
assertAPIError(router.t, w, errCode)
}
func putBucketTaggingBase(router *routerMock, namespace, bktName string, tagging []byte) *httptest.ResponseRecorder {
queries := url.Values{}
queries.Add(s3middleware.TaggingQuery, "")
w, r := httptest.NewRecorder(), httptest.NewRequest(http.MethodPut, "/"+bktName, bytes.NewBuffer(tagging))
r.URL.RawQuery = queries.Encode()
r.Header.Set(FrostfsNamespaceHeader, namespace)
router.ServeHTTP(w, r)
return w
}
func getObject(router *routerMock, namespace, bktName, objName string) handlerResult {
w := getObjectBase(router, namespace, bktName, objName)
resp := readResponse(router.t, w)
require.Equal(router.t, s3middleware.GetObjectOperation, resp.Method)
return resp
}
func getObjectErr(router *routerMock, namespace, bktName, objName string, errCode apiErrors.ErrorCode) {
w := getObjectBase(router, namespace, bktName, objName)
assertAPIError(router.t, w, errCode)
}
func getObjectBase(router *routerMock, namespace, bktName, objName string) *httptest.ResponseRecorder {
w, r := httptest.NewRecorder(), httptest.NewRequest(http.MethodGet, "/"+bktName+"/"+objName, nil)
r.Header.Set(FrostfsNamespaceHeader, namespace)
router.ServeHTTP(w, r)
return w
@ -596,11 +761,11 @@ func TestOwnerIDRetrieving(t *testing.T) {
createBucket(chiRouter, ns, bktName)
resp := putObject(chiRouter, ns, bktName, objName)
resp := putObject(chiRouter, ns, bktName, objName, nil)
require.NotEqual(t, "anon", resp.ReqInfo.User)
chiRouter.cfg.Center.(*centerMock).anon = true
resp = putObject(chiRouter, ns, bktName, objName)
resp = putObject(chiRouter, ns, bktName, objName, nil)
require.Equal(t, "anon", resp.ReqInfo.User)
}
@ -618,7 +783,7 @@ func TestBillingMetrics(t *testing.T) {
require.Equal(t, 1, dump.Requests[0].Requests)
chiRouter.cfg.Center.(*centerMock).anon = true
putObject(chiRouter, ns, bktName, objName)
putObject(chiRouter, ns, bktName, objName, nil)
dump = chiRouter.cfg.Metrics.UsersAPIStats().DumpMetrics()
require.Len(t, dump.Requests, 1)
require.Equal(t, "anon", dump.Requests[0].User)

3
cmd/s3-gw/app.go
View file

@ -688,6 +688,9 @@ func (a *App) Serve(ctx context.Context) {
FrostfsID: a.frostfsid,
FrostFSIDValidation: a.settings.frostfsidValidation,
XMLDecoder: a.settings,
Tagging: a.obj,
}
chiRouter := api.NewRouter(cfg)