TrueCloudLab/frostfs-s3-gw
17
3
Fork 19
Code Issues 29 Pull requests 7 Projects Releases 18 Packages Activity Actions

[#357] Add check of request and resource tags

Browse source 
Signed-off-by: Marina Biryukova <m.biryukova@yadro.com>
This commit is contained in:
Marina Biryukova 2024-04-10 09:41:07 +03:00 committed by Alexey Vanin
parent 9f29fcbd52
commit 3ff027587c
24 changed files with 506 additions and 155 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

8
api/data/info.go
View file

@ -83,6 +83,14 @@ type (
ExposeHeaders []string `xml:"ExposeHeader" json:"ExposeHeaders"`
MaxAgeSeconds int `xml:"MaxAgeSeconds,omitempty" json:"MaxAgeSeconds,omitempty"`
}
// ObjectVersion stores object version info.
ObjectVersion struct {
BktInfo *BucketInfo
ObjectName string
VersionID string
NoErrorOnDeleteMarker bool
}
)
// NotificationInfoFromObject creates new NotificationInfo from ObjectInfo.

30
api/data/tagging.go Normal file
View file

@ -0,0 +1,30 @@
package data
import "encoding/xml"
// Tagging contains tag set.
type Tagging struct {
XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Tagging"`
TagSet []Tag `xml:"TagSet>Tag"`
}
// Tag is an AWS key-value tag.
type Tag struct {
Key string
Value string
}
type GetObjectTaggingParams struct {
ObjectVersion *ObjectVersion
// NodeVersion can be nil. If not nil we save one request to tree service.
NodeVersion *NodeVersion // optional
}
type PutObjectTaggingParams struct {
ObjectVersion *ObjectVersion
TagSet map[string]string
// NodeVersion can be nil. If not nil we save one request to tree service.
NodeVersion *NodeVersion // optional
}

8
api/handler/copy.go
View file

@ -168,8 +168,8 @@ func (h *handler) CopyObjectHandler(w http.ResponseWriter, r *http.Request) {
return
}
} else {
tagPrm := &layer.GetObjectTaggingParams{
ObjectVersion: &layer.ObjectVersion{
tagPrm := &data.GetObjectTaggingParams{
ObjectVersion: &data.ObjectVersion{
BktInfo: srcObjPrm.BktInfo,
ObjectName: srcObject,
VersionID: srcObjInfo.VersionID(),
@ -259,8 +259,8 @@ func (h *handler) CopyObjectHandler(w http.ResponseWriter, r *http.Request) {
}
if tagSet != nil {
tagPrm := &layer.PutObjectTaggingParams{
ObjectVersion: &layer.ObjectVersion{
tagPrm := &data.PutObjectTaggingParams{
ObjectVersion: &data.ObjectVersion{
BktInfo: dstBktInfo,
ObjectName: reqInfo.ObjectName,
VersionID: dstObjInfo.VersionID(),

13
api/handler/copy_test.go
View file

@ -11,9 +11,11 @@ import (
"testing"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer/encryption"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/middleware"
"github.com/stretchr/testify/require"
)
@ -289,23 +291,24 @@ func copyObject(hc *handlerContext, bktName, fromObject, toObject string, copyMe
}
func putObjectTagging(t *testing.T, tc *handlerContext, bktName, objName string, tags map[string]string) {
body := &Tagging{
TagSet: make([]Tag, 0, len(tags)),
body := &data.Tagging{
TagSet: make([]data.Tag, 0, len(tags)),
}
for key, val := range tags {
body.TagSet = append(body.TagSet, Tag{
body.TagSet = append(body.TagSet, data.Tag{
Key: key,
Value: val,
})
}
w, r := prepareTestRequest(tc, bktName, objName, body)
middleware.GetReqInfo(r.Context()).Tagging = body
tc.Handler().PutObjectTaggingHandler(w, r)
assertStatus(t, w, http.StatusOK)
}
func getObjectTagging(t *testing.T, tc *handlerContext, bktName, objName, version string) *Tagging {
func getObjectTagging(t *testing.T, tc *handlerContext, bktName, objName, version string) *data.Tagging {
query := make(url.Values)
query.Add(api.QueryVersionID, version)
@ -313,7 +316,7 @@ func getObjectTagging(t *testing.T, tc *handlerContext, bktName, objName, versio
tc.Handler().GetObjectTaggingHandler(w, r)
assertStatus(t, w, http.StatusOK)
tagging := &Tagging{}
tagging := &data.Tagging{}
err := xml.NewDecoder(w.Result().Body).Decode(tagging)
require.NoError(t, err)
return tagging

2
api/handler/get.go
View file

@ -184,7 +184,7 @@ func (h *handler) GetObjectHandler(w http.ResponseWriter, r *http.Request) {
return
}
t := &layer.ObjectVersion{
t := &data.ObjectVersion{
BktInfo: bktInfo,
ObjectName: info.Name,
VersionID: info.VersionID(),

2
api/handler/head.go
View file

@ -70,7 +70,7 @@ func (h *handler) HeadObjectHandler(w http.ResponseWriter, r *http.Request) {
return
}
t := &layer.ObjectVersion{
t := &data.ObjectVersion{
BktInfo: bktInfo,
ObjectName: info.Name,
VersionID: info.VersionID(),

8
api/handler/locking.go
View file

@ -133,7 +133,7 @@ func (h *handler) PutObjectLegalHoldHandler(w http.ResponseWriter, r *http.Reque
}
p := &layer.PutLockInfoParams{
ObjVersion: &layer.ObjectVersion{
ObjVersion: &data.ObjectVersion{
BktInfo: bktInfo,
ObjectName: reqInfo.ObjectName,
VersionID: reqInfo.URL.Query().Get(api.QueryVersionID),
@ -172,7 +172,7 @@ func (h *handler) GetObjectLegalHoldHandler(w http.ResponseWriter, r *http.Reque
return
}
p := &layer.ObjectVersion{
p := &data.ObjectVersion{
BktInfo: bktInfo,
ObjectName: reqInfo.ObjectName,
VersionID: reqInfo.URL.Query().Get(api.QueryVersionID),
@ -221,7 +221,7 @@ func (h *handler) PutObjectRetentionHandler(w http.ResponseWriter, r *http.Reque
}
p := &layer.PutLockInfoParams{
ObjVersion: &layer.ObjectVersion{
ObjVersion: &data.ObjectVersion{
BktInfo: bktInfo,
ObjectName: reqInfo.ObjectName,
VersionID: reqInfo.URL.Query().Get(api.QueryVersionID),
@ -256,7 +256,7 @@ func (h *handler) GetObjectRetentionHandler(w http.ResponseWriter, r *http.Reque
return
}
p := &layer.ObjectVersion{
p := &data.ObjectVersion{
BktInfo: bktInfo,
ObjectName: reqInfo.ObjectName,
VersionID: reqInfo.URL.Query().Get(api.QueryVersionID),

4
api/handler/multipart_upload.go
View file

@ -487,8 +487,8 @@ func (h *handler) completeMultipartUpload(r *http.Request, c *layer.CompleteMult
objInfo := extendedObjInfo.ObjectInfo
if len(uploadData.TagSet) != 0 {
tagPrm := &layer.PutObjectTaggingParams{
ObjectVersion: &layer.ObjectVersion{
tagPrm := &data.PutObjectTaggingParams{
ObjectVersion: &data.ObjectVersion{
BktInfo: bktInfo,
ObjectName: objInfo.Name,
VersionID: objInfo.VersionID(),

17
api/handler/put.go
View file

@ -307,8 +307,8 @@ func (h *handler) PutObjectHandler(w http.ResponseWriter, r *http.Request) {
}
if tagSet != nil {
tagPrm := &layer.PutObjectTaggingParams{
ObjectVersion: &layer.ObjectVersion{
tagPrm := &data.PutObjectTaggingParams{
ObjectVersion: &data.ObjectVersion{
BktInfo: bktInfo,
ObjectName: objInfo.Name,
VersionID: objInfo.VersionID(),
@ -483,7 +483,12 @@ func (h *handler) PostObject(w http.ResponseWriter, r *http.Request) {
if tagging := auth.MultipartFormValue(r, "tagging"); tagging != "" {
buffer := bytes.NewBufferString(tagging)
tagSet, err = h.readTagSet(buffer)
tags := new(data.Tagging)
if err = h.cfg.NewXMLDecoder(buffer).Decode(tags); err != nil {
h.logAndSendError(w, "could not decode tag set", reqInfo, errors.GetAPIError(errors.ErrMalformedXML))
return
}
tagSet, err = h.readTagSet(tags)
if err != nil {
h.logAndSendError(w, "could not read tag set", reqInfo, err)
return
@ -574,8 +579,8 @@ func (h *handler) PostObject(w http.ResponseWriter, r *http.Request) {
}
if tagSet != nil {
tagPrm := &layer.PutObjectTaggingParams{
ObjectVersion: &layer.ObjectVersion{
tagPrm := &data.PutObjectTaggingParams{
ObjectVersion: &data.ObjectVersion{
BktInfo: bktInfo,
ObjectName: objInfo.Name,
VersionID: objInfo.VersionID(),
@ -789,7 +794,7 @@ func parseTaggingHeader(header http.Header) (map[string]string, error) {
}
tagSet = make(map[string]string, len(queries))
for k, v := range queries {
tag := Tag{Key: k, Value: v[0]}
tag := data.Tag{Key: k, Value: v[0]}
if err = checkTag(tag); err != nil {
return nil, err
}

12
api/handler/response.go
View file

@ -185,12 +185,6 @@ type VersioningConfiguration struct {
MfaDelete string `xml:"MfaDelete,omitempty"`
}
// Tagging contains tag set.
type Tagging struct {
XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Tagging"`
TagSet []Tag `xml:"TagSet>Tag"`
}
// PostResponse contains result of posting object.
type PostResponse struct {
Bucket string `xml:"Bucket"`
@ -198,12 +192,6 @@ type PostResponse struct {
ETag string `xml:"Etag"`
}
// Tag is an AWS key-value tag.
type Tag struct {
Key string
Value string
}
// MarshalXML -- StringMap marshals into XML.
func (s StringMap) MarshalXML(e *xml.Encoder, start xml.StartElement) error {
tokens := []xml.Token{start}

33
api/handler/tagging.go
View file

@ -1,7 +1,6 @@
package handler
import (
"io"
"net/http"
"sort"
"strings"
@ -10,7 +9,6 @@ import (
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/middleware"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs"
"go.uber.org/zap"
@ -28,7 +26,7 @@ func (h *handler) PutObjectTaggingHandler(w http.ResponseWriter, r *http.Request
ctx := r.Context()
reqInfo := middleware.GetReqInfo(ctx)
tagSet, err := h.readTagSet(r.Body)
tagSet, err := h.readTagSet(reqInfo.Tagging)
if err != nil {
h.logAndSendError(w, "could not read tag set", reqInfo, err)
return
@ -40,8 +38,8 @@ func (h *handler) PutObjectTaggingHandler(w http.ResponseWriter, r *http.Request
return
}
tagPrm := &layer.PutObjectTaggingParams{
ObjectVersion: &layer.ObjectVersion{
tagPrm := &data.PutObjectTaggingParams{
ObjectVersion: &data.ObjectVersion{
BktInfo: bktInfo,
ObjectName: reqInfo.ObjectName,
VersionID: reqInfo.URL.Query().Get(api.QueryVersionID),
@ -87,8 +85,8 @@ func (h *handler) GetObjectTaggingHandler(w http.ResponseWriter, r *http.Request
return
}
tagPrm := &layer.GetObjectTaggingParams{
ObjectVersion: &layer.ObjectVersion{
tagPrm := &data.GetObjectTaggingParams{
ObjectVersion: &data.ObjectVersion{
BktInfo: bktInfo,
ObjectName: reqInfo.ObjectName,
VersionID: reqInfo.URL.Query().Get(api.QueryVersionID),
@ -119,7 +117,7 @@ func (h *handler) DeleteObjectTaggingHandler(w http.ResponseWriter, r *http.Requ
return
}
p := &layer.ObjectVersion{
p := &data.ObjectVersion{
BktInfo: bktInfo,
ObjectName: reqInfo.ObjectName,
VersionID: reqInfo.URL.Query().Get(api.QueryVersionID),
@ -152,7 +150,7 @@ func (h *handler) DeleteObjectTaggingHandler(w http.ResponseWriter, r *http.Requ
func (h *handler) PutBucketTaggingHandler(w http.ResponseWriter, r *http.Request) {
reqInfo := middleware.GetReqInfo(r.Context())
tagSet, err := h.readTagSet(r.Body)
tagSet, err := h.readTagSet(reqInfo.Tagging)
if err != nil {
h.logAndSendError(w, "could not read tag set", reqInfo, err)
return
@ -207,12 +205,7 @@ func (h *handler) DeleteBucketTaggingHandler(w http.ResponseWriter, r *http.Requ
w.WriteHeader(http.StatusNoContent)
}
func (h *handler) readTagSet(reader io.Reader) (map[string]string, error) {
tagging := new(Tagging)
if err := h.cfg.NewXMLDecoder(reader).Decode(tagging); err != nil {
return nil, errors.GetAPIError(errors.ErrMalformedXML)
}
func (h *handler) readTagSet(tagging *data.Tagging) (map[string]string, error) {
if err := checkTagSet(tagging.TagSet); err != nil {
return nil, err
}
@ -228,10 +221,10 @@ func (h *handler) readTagSet(reader io.Reader) (map[string]string, error) {
return tagSet, nil
}
func encodeTagging(tagSet map[string]string) *Tagging {
tagging := &Tagging{}
func encodeTagging(tagSet map[string]string) *data.Tagging {
tagging := &data.Tagging{}
for k, v := range tagSet {
tagging.TagSet = append(tagging.TagSet, Tag{Key: k, Value: v})
tagging.TagSet = append(tagging.TagSet, data.Tag{Key: k, Value: v})
}
sort.Slice(tagging.TagSet, func(i, j int) bool {
return tagging.TagSet[i].Key < tagging.TagSet[j].Key
@ -240,7 +233,7 @@ func encodeTagging(tagSet map[string]string) *Tagging {
return tagging
}
func checkTagSet(tagSet []Tag) error {
func checkTagSet(tagSet []data.Tag) error {
if len(tagSet) > maxTags {
return errors.GetAPIError(errors.ErrInvalidTagsSizeExceed)
}
@ -254,7 +247,7 @@ func checkTagSet(tagSet []Tag) error {
return nil
}
func checkTag(tag Tag) error {
func checkTag(tag data.Tag) error {
if len(tag.Key) < 1 || len(tag.Key) > keyTagMaxLength {
return errors.GetAPIError(errors.ErrInvalidTagKey)
}

41
api/handler/tagging_test.go
View file

@ -5,7 +5,9 @@ import (
"strings"
"testing"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
apiErrors "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/middleware"
"github.com/stretchr/testify/require"
)
@ -20,23 +22,23 @@ func TestTagsValidity(t *testing.T) {
}
for _, tc := range []struct {
tag Tag
tag data.Tag
valid bool
}{
{tag: Tag{}, valid: false},
{tag: Tag{Key: "", Value: "1"}, valid: false},
{tag: Tag{Key: "aws:key", Value: "val"}, valid: false},
{tag: Tag{Key: "key~", Value: "val"}, valid: false},
{tag: Tag{Key: "key\\", Value: "val"}, valid: false},
{tag: Tag{Key: "key?", Value: "val"}, valid: false},
{tag: Tag{Key: sbKey.String() + "b", Value: "val"}, valid: false},
{tag: Tag{Key: "key", Value: sbValue.String() + "b"}, valid: false},
{tag: data.Tag{}, valid: false},
{tag: data.Tag{Key: "", Value: "1"}, valid: false},
{tag: data.Tag{Key: "aws:key", Value: "val"}, valid: false},
{tag: data.Tag{Key: "key~", Value: "val"}, valid: false},
{tag: data.Tag{Key: "key\\", Value: "val"}, valid: false},
{tag: data.Tag{Key: "key?", Value: "val"}, valid: false},
{tag: data.Tag{Key: sbKey.String() + "b", Value: "val"}, valid: false},
{tag: data.Tag{Key: "key", Value: sbValue.String() + "b"}, valid: false},
{tag: Tag{Key: sbKey.String(), Value: "val"}, valid: true},
{tag: Tag{Key: "key", Value: sbValue.String()}, valid: true},
{tag: Tag{Key: "k e y", Value: "v a l"}, valid: true},
{tag: Tag{Key: "12345", Value: "1234"}, valid: true},
{tag: Tag{Key: allowedTagChars, Value: allowedTagChars}, valid: true},
{tag: data.Tag{Key: sbKey.String(), Value: "val"}, valid: true},
{tag: data.Tag{Key: "key", Value: sbValue.String()}, valid: true},
{tag: data.Tag{Key: "k e y", Value: "v a l"}, valid: true},
{tag: data.Tag{Key: "12345", Value: "1234"}, valid: true},
{tag: data.Tag{Key: allowedTagChars, Value: allowedTagChars}, valid: true},
} {
err := checkTag(tc.tag)
if tc.valid {
@ -55,13 +57,13 @@ func TestPutObjectTaggingCheckUniqueness(t *testing.T) {
for _, tc := range []struct {
name string
body *Tagging
body *data.Tagging
error bool
}{
{
name: "Two tags with unique keys",
body: &Tagging{
TagSet: []Tag{
body: &data.Tagging{
TagSet: []data.Tag{
{
Key: "key-1",
Value: "val-1",
@ -76,8 +78,8 @@ func TestPutObjectTaggingCheckUniqueness(t *testing.T) {
},
{
name: "Two tags with the same keys",
body: &Tagging{
TagSet: []Tag{
body: &data.Tagging{
TagSet: []data.Tag{
{
Key: "key-1",
Value: "val-1",
@ -93,6 +95,7 @@ func TestPutObjectTaggingCheckUniqueness(t *testing.T) {
} {
t.Run(tc.name, func(t *testing.T) {
w, r := prepareTestRequest(hc, bktName, objName, tc.body)
middleware.GetReqInfo(r.Context()).Tagging = tc.body
hc.Handler().PutObjectTaggingHandler(w, r)
if tc.error {
assertS3Error(t, w, apiErrors.GetAPIError(apiErrors.ErrInvalidTagKeyUniqueness))

2
api/layer/compound.go
View file

@ -9,7 +9,7 @@ import (
s3errors "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
)
func (n *layer) GetObjectTaggingAndLock(ctx context.Context, objVersion *ObjectVersion, nodeVersion *data.NodeVersion) (map[string]string, data.LockInfo, error) {
func (n *layer) GetObjectTaggingAndLock(ctx context.Context, objVersion *data.ObjectVersion, nodeVersion *data.NodeVersion) (map[string]string, data.LockInfo, error) {
var err error
owner := n.BearerOwner(ctx)

22
api/layer/layer.go
View file

@ -98,14 +98,6 @@ type (
VersionID string
}
// ObjectVersion stores object version info.
ObjectVersion struct {
BktInfo *data.BucketInfo
ObjectName string
VersionID string
NoErrorOnDeleteMarker bool
}
// RangeParams stores range header request parameters.
RangeParams struct {
Start uint64
@ -245,16 +237,16 @@ type (
GetObjectInfo(ctx context.Context, p *HeadObjectParams) (*data.ObjectInfo, error)
GetExtendedObjectInfo(ctx context.Context, p *HeadObjectParams) (*data.ExtendedObjectInfo, error)
GetLockInfo(ctx context.Context, obj *ObjectVersion) (*data.LockInfo, error)
GetLockInfo(ctx context.Context, obj *data.ObjectVersion) (*data.LockInfo, error)
PutLockInfo(ctx context.Context, p *PutLockInfoParams) error
GetBucketTagging(ctx context.Context, bktInfo *data.BucketInfo) (map[string]string, error)
PutBucketTagging(ctx context.Context, bktInfo *data.BucketInfo, tagSet map[string]string) error
DeleteBucketTagging(ctx context.Context, bktInfo *data.BucketInfo) error
GetObjectTagging(ctx context.Context, p *GetObjectTaggingParams) (string, map[string]string, error)
PutObjectTagging(ctx context.Context, p *PutObjectTaggingParams) (*data.NodeVersion, error)
DeleteObjectTagging(ctx context.Context, p *ObjectVersion) (*data.NodeVersion, error)
GetObjectTagging(ctx context.Context, p *data.GetObjectTaggingParams) (string, map[string]string, error)
PutObjectTagging(ctx context.Context, p *data.PutObjectTaggingParams) (*data.NodeVersion, error)
DeleteObjectTagging(ctx context.Context, p *data.ObjectVersion) (*data.NodeVersion, error)
PutObject(ctx context.Context, p *PutObjectParams) (*data.ExtendedObjectInfo, error)
@ -280,7 +272,7 @@ type (
// Compound methods for optimizations
// GetObjectTaggingAndLock unifies GetObjectTagging and GetLock methods in single tree service invocation.
GetObjectTaggingAndLock(ctx context.Context, p *ObjectVersion, nodeVersion *data.NodeVersion) (map[string]string, data.LockInfo, error)
GetObjectTaggingAndLock(ctx context.Context, p *data.ObjectVersion, nodeVersion *data.NodeVersion) (map[string]string, data.LockInfo, error)
}
)
@ -766,7 +758,7 @@ func isNotFoundError(err error) bool {
}
func (n *layer) getNodeVersionToDelete(ctx context.Context, bkt *data.BucketInfo, obj *VersionedObject) (*data.NodeVersion, error) {
objVersion := &ObjectVersion{
objVersion := &data.ObjectVersion{
BktInfo: bkt,
ObjectName: obj.Name,
VersionID: obj.VersionID,
@ -777,7 +769,7 @@ func (n *layer) getNodeVersionToDelete(ctx context.Context, bkt *data.BucketInfo
}
func (n *layer) getLastNodeVersion(ctx context.Context, bkt *data.BucketInfo, obj *VersionedObject) (*data.NodeVersion, error) {
objVersion := &ObjectVersion{
objVersion := &data.ObjectVersion{
BktInfo: bkt,
ObjectName: obj.Name,
VersionID: "",

2
api/layer/locking_test.go
View file

@ -20,7 +20,7 @@ func TestObjectLockAttributes(t *testing.T) {
obj := tc.putObject([]byte("content obj1 v1"))
p := &PutLockInfoParams{
ObjVersion: &ObjectVersion{
ObjVersion: &data.ObjectVersion{
BktInfo: tc.bktInfo,
ObjectName: obj.Name,
VersionID: obj.VersionID(),

2
api/layer/object.go
View file

@ -321,7 +321,7 @@ func (n *layer) PutObject(ctx context.Context, p *PutObjectParams) (*data.Extend
if p.Lock != nil && (p.Lock.Retention != nil || p.Lock.LegalHold != nil) {
putLockInfoPrms := &PutLockInfoParams{
ObjVersion: &ObjectVersion{
ObjVersion: &data.ObjectVersion{
BktInfo: p.BktInfo,
ObjectName: p.Object,
VersionID: id.EncodeToString(),

8
api/layer/system_object.go
View file

@ -20,7 +20,7 @@ const (
)
type PutLockInfoParams struct {
ObjVersion *ObjectVersion
ObjVersion *data.ObjectVersion
NewLock *data.ObjectLock
CopiesNumbers []uint32
NodeVersion *data.NodeVersion // optional
@ -100,7 +100,7 @@ func (n *layer) PutLockInfo(ctx context.Context, p *PutLockInfoParams) (err erro
return nil
}
func (n *layer) getNodeVersionFromCacheOrFrostfs(ctx context.Context, objVersion *ObjectVersion) (nodeVersion *data.NodeVersion, err error) {
func (n *layer) getNodeVersionFromCacheOrFrostfs(ctx context.Context, objVersion *data.ObjectVersion) (nodeVersion *data.NodeVersion, err error) {
// check cache if node version is stored inside extendedObjectVersion
nodeVersion = n.getNodeVersionFromCache(n.BearerOwner(ctx), objVersion)
if nodeVersion == nil {
@ -129,7 +129,7 @@ func (n *layer) putLockObject(ctx context.Context, bktInfo *data.BucketInfo, obj
return id, err
}
func (n *layer) GetLockInfo(ctx context.Context, objVersion *ObjectVersion) (*data.LockInfo, error) {
func (n *layer) GetLockInfo(ctx context.Context, objVersion *data.ObjectVersion) (*data.LockInfo, error) {
owner := n.BearerOwner(ctx)
if lockInfo := n.cache.GetLockInfo(owner, lockObjectKey(objVersion)); lockInfo != nil {
return lockInfo, nil
@ -185,7 +185,7 @@ func (n *layer) getCORS(ctx context.Context, bkt *data.BucketInfo) (*data.CORSCo
return cors, nil
}
func lockObjectKey(objVersion *ObjectVersion) string {
func lockObjectKey(objVersion *data.ObjectVersion) string {
// todo reconsider forming name since versionID can be "null" or ""
return ".lock." + objVersion.BktInfo.CID.EncodeToString() + "." + objVersion.ObjectName + "." + objVersion.VersionID
}

27
api/layer/tagging.go
View file

@ -14,22 +14,7 @@ import (
"go.uber.org/zap"
)
type GetObjectTaggingParams struct {
ObjectVersion *ObjectVersion
// NodeVersion can be nil. If not nil we save one request to tree service.
NodeVersion *data.NodeVersion // optional
}
type PutObjectTaggingParams struct {
ObjectVersion *ObjectVersion
TagSet map[string]string
// NodeVersion can be nil. If not nil we save one request to tree service.
NodeVersion *data.NodeVersion // optional
}
func (n *layer) GetObjectTagging(ctx context.Context, p *GetObjectTaggingParams) (string, map[string]string, error) {
func (n *layer) GetObjectTagging(ctx context.Context, p *data.GetObjectTaggingParams) (string, map[string]string, error) {
var err error
owner := n.BearerOwner(ctx)
@ -65,7 +50,7 @@ func (n *layer) GetObjectTagging(ctx context.Context, p *GetObjectTaggingParams)
return p.ObjectVersion.VersionID, tags, nil
}
func (n *layer) PutObjectTagging(ctx context.Context, p *PutObjectTaggingParams) (nodeVersion *data.NodeVersion, err error) {
func (n *layer) PutObjectTagging(ctx context.Context, p *data.PutObjectTaggingParams) (nodeVersion *data.NodeVersion, err error) {
nodeVersion = p.NodeVersion
if nodeVersion == nil {
nodeVersion, err = n.getNodeVersionFromCacheOrFrostfs(ctx, p.ObjectVersion)
@ -88,7 +73,7 @@ func (n *layer) PutObjectTagging(ctx context.Context, p *PutObjectTaggingParams)
return nodeVersion, nil
}
func (n *layer) DeleteObjectTagging(ctx context.Context, p *ObjectVersion) (*data.NodeVersion, error) {
func (n *layer) DeleteObjectTagging(ctx context.Context, p *data.ObjectVersion) (*data.NodeVersion, error) {
version, err := n.getNodeVersion(ctx, p)
if err != nil {
return nil, err
@ -142,7 +127,7 @@ func (n *layer) DeleteBucketTagging(ctx context.Context, bktInfo *data.BucketInf
return n.treeService.DeleteBucketTagging(ctx, bktInfo)
}
func objectTaggingCacheKey(p *ObjectVersion) string {
func objectTaggingCacheKey(p *data.ObjectVersion) string {
return ".tagset." + p.BktInfo.CID.EncodeToString() + "." + p.ObjectName + "." + p.VersionID
}
@ -150,7 +135,7 @@ func bucketTaggingCacheKey(cnrID cid.ID) string {
return ".tagset." + cnrID.EncodeToString()
}
func (n *layer) getNodeVersion(ctx context.Context, objVersion *ObjectVersion) (*data.NodeVersion, error) {
func (n *layer) getNodeVersion(ctx context.Context, objVersion *data.ObjectVersion) (*data.NodeVersion, error) {
var err error
var version *data.NodeVersion
@ -188,7 +173,7 @@ func (n *layer) getNodeVersion(ctx context.Context, objVersion *ObjectVersion) (
return version, err
}
func (n *layer) getNodeVersionFromCache(owner user.ID, o *ObjectVersion) *data.NodeVersion {
func (n *layer) getNodeVersionFromCache(owner user.ID, o *data.ObjectVersion) *data.NodeVersion {
if len(o.VersionID) == 0 || o.VersionID == data.UnversionedObjectVersionID {
return nil
}

152
api/middleware/policy.go
View file

@ -3,12 +3,16 @@ package middleware
import (
"context"
"crypto/elliptic"
"encoding/xml"
"fmt"
"io"
"net/http"
"net/url"
"strings"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
apiErr "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
frostfsErrors "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/frostfs/errors"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs"
"git.frostfs.info/TrueCloudLab/policy-engine/pkg/chain"
"git.frostfs.info/TrueCloudLab/policy-engine/pkg/engine"
@ -25,8 +29,22 @@ const (
QueryPrefix = "prefix"
QueryDelimiter = "delimiter"
QueryMaxKeys = "max-keys"
amzTagging = "x-amz-tagging"
)
// At the beginning of these operations resources haven't yet been created.
var withoutResourceOps = []string{
CreateBucketOperation,
CreateMultipartUploadOperation,
AbortMultipartUploadOperation,
CompleteMultipartUploadOperation,
UploadPartOperation,
UploadPartCopyOperation,
ListPartsOperation,
PutObjectOperation,
CopyObjectOperation,
}
type PolicySettings interface {
PolicyDenyByDefault() bool
ACLEnabled() bool
@ -36,6 +54,15 @@ type FrostFSIDInformer interface {
GetUserGroupIDs(userHash util.Uint160) ([]string, error)
}
type XMLDecoder interface {
NewXMLDecoder(io.Reader) *xml.Decoder
}
type ResourceTagging interface {
GetBucketTagging(ctx context.Context, bktInfo *data.BucketInfo) (map[string]string, error)
GetObjectTagging(ctx context.Context, p *data.GetObjectTaggingParams) (string, map[string]string, error)
}
// BucketResolveFunc is a func to resolve bucket info by name.
type BucketResolveFunc func(ctx context.Context, bucket string) (*data.BucketInfo, error)
@ -46,6 +73,8 @@ type PolicyConfig struct {
Domains []string
Log *zap.Logger
BucketResolver BucketResolveFunc
Decoder XMLDecoder
Tagging ResourceTagging
}
func PolicyCheck(cfg PolicyConfig) Func {
@ -54,6 +83,7 @@ func PolicyCheck(cfg PolicyConfig) Func {
ctx := r.Context()
if err := policyCheck(r, cfg); err != nil {
reqLogOrDefault(ctx, cfg.Log).Error(logs.PolicyValidationFailed, zap.Error(err))
err = frostfsErrors.UnwrapErr(err)
if _, wrErr := WriteErrorResponse(w, GetReqInfo(ctx), err); wrErr != nil {
reqLogOrDefault(ctx, cfg.Log).Error(logs.FailedToWriteResponse, zap.Error(wrErr))
}
@ -67,7 +97,7 @@ func PolicyCheck(cfg PolicyConfig) Func {
func policyCheck(r *http.Request, cfg PolicyConfig) error {
reqType, bktName, objName := getBucketObject(r, cfg.Domains)
req, err := getPolicyRequest(r, cfg.FrostfsID, reqType, bktName, objName, cfg.Log)
req, err := getPolicyRequest(r, cfg, reqType, bktName, objName)
if err != nil {
return err
}
@ -115,7 +145,7 @@ func policyCheck(r *http.Request, cfg PolicyConfig) error {
return nil
}
func getPolicyRequest(r *http.Request, frostfsid FrostFSIDInformer, reqType ReqType, bktName string, objName string, log *zap.Logger) (*testutil.Request, error) {
func getPolicyRequest(r *http.Request, cfg PolicyConfig, reqType ReqType, bktName string, objName string) (*testutil.Request, error) {
var (
owner string
groups []string
@ -130,7 +160,7 @@ func getPolicyRequest(r *http.Request, frostfsid FrostFSIDInformer, reqType ReqT
}
owner = pk.Address()
groups, err = frostfsid.GetUserGroupIDs(pk.GetScriptHash())
groups, err = cfg.FrostfsID.GetUserGroupIDs(pk.GetScriptHash())
if err != nil {
return nil, fmt.Errorf("get group ids: %w", err)
}
@ -145,9 +175,12 @@ func getPolicyRequest(r *http.Request, frostfsid FrostFSIDInformer, reqType ReqT
res = fmt.Sprintf(s3.ResourceFormatS3Bucket, bktName)
}
properties := determineProperties(ctx, reqType, op, owner, groups)
properties, err := determineProperties(r, cfg.Decoder, cfg.BucketResolver, cfg.Tagging, reqType, op, bktName, objName, owner, groups)
if err != nil {
return nil, fmt.Errorf("determine properties: %w", err)
}
reqLogOrDefault(r.Context(), log).Debug(logs.PolicyRequest, zap.String("action", op),
reqLogOrDefault(r.Context(), cfg.Log).Debug(logs.PolicyRequest, zap.String("action", op),
zap.String("resource", res), zap.Any("properties", properties))
return testutil.NewRequest(op, testutil.NewResource(res, nil), properties), nil
@ -376,12 +409,13 @@ func determineGeneralOperation(r *http.Request) string {
return "UnmatchedOperation"
}
func determineProperties(ctx context.Context, reqType ReqType, op, owner string, groups []string) map[string]string {
func determineProperties(r *http.Request, decoder XMLDecoder, resolver BucketResolveFunc, tagging ResourceTagging, reqType ReqType,
op, bktName, objName, owner string, groups []string) (map[string]string, error) {
res := map[string]string{
s3.PropertyKeyOwner: owner,
common.PropertyKeyFrostFSIDGroupID: chain.FormCondSliceContainsValue(groups),
}
queries := GetReqInfo(ctx).URL.Query()
queries := GetReqInfo(r.Context()).URL.Query()
if reqType == objectType {
if versionID := queries.Get(QueryVersionID); len(versionID) > 0 {
@ -402,5 +436,107 @@ func determineProperties(ctx context.Context, reqType ReqType, op, owner string,
}
}
return res
tags, err := determineTags(r, decoder, resolver, tagging, reqType, op, bktName, objName, queries.Get(QueryVersionID))
if err != nil {
return nil, fmt.Errorf("determine tags: %w", err)
}
for k, v := range tags {
res[k] = v
}
return res, nil
}
func determineTags(r *http.Request, decoder XMLDecoder, resolver BucketResolveFunc, tagging ResourceTagging, reqType ReqType,
op, bktName, objName, versionID string) (map[string]string, error) {
res, err := determineRequestTags(r, decoder, op)
if err != nil {
return nil, fmt.Errorf("determine request tags: %w", err)
}
tags, err := determineResourceTags(r.Context(), reqType, op, bktName, objName, versionID, resolver, tagging)
if err != nil {
return nil, fmt.Errorf("determine resource tags: %w", err)
}
for k, v := range tags {
res[k] = v
}
return res, nil
}
func determineRequestTags(r *http.Request, decoder XMLDecoder, op string) (map[string]string, error) {
tags := make(map[string]string)
if strings.HasSuffix(op, PutObjectTaggingOperation) || strings.HasSuffix(op, PutBucketTaggingOperation) {
tagging := new(data.Tagging)
if err := decoder.NewXMLDecoder(r.Body).Decode(tagging); err != nil {
return nil, apiErr.GetAPIError(apiErr.ErrMalformedXML)
}
GetReqInfo(r.Context()).Tagging = tagging
for _, tag := range tagging.TagSet {
tags[fmt.Sprintf(s3.PropertyKeyFormatRequestTag, tag.Key)] = tag.Value
}
}
if tagging := r.Header.Get(amzTagging); len(tagging) > 0 {
queries, err := url.ParseQuery(tagging)
if err != nil {
return nil, apiErr.GetAPIError(apiErr.ErrInvalidArgument)
}
for key := range queries {
tags[fmt.Sprintf(s3.PropertyKeyFormatRequestTag, key)] = queries.Get(key)
}
}
return tags, nil
}
func determineResourceTags(ctx context.Context, reqType ReqType, op, bktName, objName, versionID string, resolver BucketResolveFunc,
tagging ResourceTagging) (map[string]string, error) {
tags := make(map[string]string)
if reqType != bucketType && reqType != objectType {
return tags, nil
}
for _, withoutResOp := range withoutResourceOps {
if strings.HasSuffix(op, withoutResOp) {
return tags, nil
}
}
bktInfo, err := resolver(ctx, bktName)
if err != nil {
return nil, fmt.Errorf("get bucket info: %w", err)
}
if reqType == bucketType {
tags, err = tagging.GetBucketTagging(ctx, bktInfo)
if err != nil {
return nil, fmt.Errorf("get bucket tagging: %w", err)
}
}
if reqType == objectType {
tagPrm := &data.GetObjectTaggingParams{
ObjectVersion: &data.ObjectVersion{
BktInfo: bktInfo,
ObjectName: objName,
VersionID: versionID,
},
}
_, tags, err = tagging.GetObjectTagging(ctx, tagPrm)
if err != nil {
return nil, fmt.Errorf("get object tagging: %w", err)
}
}
res := make(map[string]string, len(tags))
for k, v := range tags {
res[fmt.Sprintf(s3.PropertyKeyFormatResourceTag, k)] = v
}
return res, nil
}

1
api/middleware/reqinfo.go
View file

@ -40,6 +40,7 @@ type (
URL *url.URL // Request url
Namespace string
User string // User owner id
Tagging *data.Tagging
tags []KeyVal // Any additional info not accommodated by above fields
}

5
api/router.go
View file

@ -121,6 +121,9 @@ type Config struct {
FrostFSIDValidation bool
PolicyChecker engine.ChainRouter
XMLDecoder s3middleware.XMLDecoder
Tagging s3middleware.ResourceTagging
}
func NewRouter(cfg Config) *chi.Mux {
@ -146,6 +149,8 @@ func NewRouter(cfg Config) *chi.Mux {
Domains: cfg.Domains,
Log: cfg.Log,
BucketResolver: cfg.Handler.ResolveBucket,
Decoder: cfg.XMLDecoder,
Tagging: cfg.Tagging,
}))
defaultRouter := chi.NewRouter()

50
api/router_mock_test.go
View file

@ -3,11 +3,13 @@ package api
import (
"context"
"encoding/json"
"errors"
"encoding/xml"
"io"
"net/http"
"testing"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
apiErrors "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/middleware"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/creds/accessbox"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer"
@ -86,6 +88,30 @@ func (f *frostFSIDMock) GetUserGroupIDs(util.Uint160) ([]string, error) {
return []string{}, nil
}
type xmlMock struct {
}
func (m *xmlMock) NewXMLDecoder(r io.Reader) *xml.Decoder {
return xml.NewDecoder(r)
}
type resourceTaggingMock struct {
bucketTags map[string]string
objectTags map[string]string
noSuchKey bool
}
func (m *resourceTaggingMock) GetBucketTagging(context.Context, *data.BucketInfo) (map[string]string, error) {
return m.bucketTags, nil
}
func (m *resourceTaggingMock) GetObjectTagging(context.Context, *data.GetObjectTaggingParams) (string, map[string]string, error) {
if m.noSuchKey {
return "", nil, apiErrors.GetAPIError(apiErrors.ErrNoSuchKey)
}
return "", m.objectTags, nil
}
type handlerMock struct {
t *testing.T
cfg *middlewareSettingsMock
@ -142,9 +168,13 @@ func (h *handlerMock) GetObjectLegalHoldHandler(http.ResponseWriter, *http.Reque
panic("implement me")
}
func (h *handlerMock) GetObjectHandler(http.ResponseWriter, *http.Request) {
//TODO implement me
panic("implement me")
func (h *handlerMock) GetObjectHandler(w http.ResponseWriter, r *http.Request) {
res := &handlerResult{
Method: middleware.GetObjectOperation,
ReqInfo: middleware.GetReqInfo(r.Context()),
}
h.writeResponse(w, res)
}
func (h *handlerMock) GetObjectAttributesHandler(http.ResponseWriter, *http.Request) {
@ -339,9 +369,13 @@ func (h *handlerMock) PutBucketObjectLockConfigHandler(http.ResponseWriter, *htt
panic("implement me")
}
func (h *handlerMock) PutBucketTaggingHandler(http.ResponseWriter, *http.Request) {
//TODO implement me
panic("implement me")
func (h *handlerMock) PutBucketTaggingHandler(w http.ResponseWriter, r *http.Request) {
res := &handlerResult{
Method: middleware.PutBucketTaggingOperation,
ReqInfo: middleware.GetReqInfo(r.Context()),
}
h.writeResponse(w, res)
}
func (h *handlerMock) PutBucketVersioningHandler(http.ResponseWriter, *http.Request) {
@ -473,7 +507,7 @@ func (h *handlerMock) ResolveBucket(ctx context.Context, name string) (*data.Buc
reqInfo := middleware.GetReqInfo(ctx)
bktInfo, ok := h.buckets[reqInfo.Namespace+name]
if !ok {
return nil, errors.New("not found")
return nil, apiErrors.GetAPIError(apiErrors.ErrNoSuchBucket)
}
return bktInfo, nil
}

207
api/router_test.go
View file

@ -1,6 +1,7 @@
package api
import (
"bytes"
"encoding/json"
"encoding/xml"
"fmt"
@ -66,6 +67,8 @@ func prepareRouter(t *testing.T) *routerMock {
PolicyChecker: policyChecker,
Domains: []string{"domain1", "domain2"},
FrostfsID: &frostFSIDMock{},
XMLDecoder: &xmlMock{},
Tagging: &resourceTaggingMock{},
}
return &routerMock{
t: t,
@ -115,7 +118,7 @@ func TestRouterObjectWithSlashes(t *testing.T) {
ns, bktName, objName := "", "dkirillov", "/fix/object"
createBucket(chiRouter, ns, bktName)
resp := putObject(chiRouter, ns, bktName, objName)
resp := putObject(chiRouter, ns, bktName, objName, nil)
require.Equal(t, objName, resp.ReqInfo.ObjectName)
}
@ -157,7 +160,7 @@ func TestRouterObjectEscaping(t *testing.T) {
},
} {
t.Run(tc.name, func(t *testing.T) {
resp := putObject(chiRouter, ns, bktName, tc.objName)
resp := putObject(chiRouter, ns, bktName, tc.objName, nil)
require.Equal(t, tc.expectedObjName, resp.ReqInfo.ObjectName)
})
}
@ -185,13 +188,13 @@ func TestPolicyChecker(t *testing.T) {
require.NoError(t, err)
// check we can access 'bucket' in default namespace
putObject(chiRouter, ns1, bktName1, objName1)
putObject(chiRouter, ns1, bktName1, objName1, nil)
// check we can access 'other-bucket' in custom namespace
putObject(chiRouter, ns2, bktName2, objName2)
putObject(chiRouter, ns2, bktName2, objName2, nil)
// check we cannot access 'bucket' in custom namespace
putObjectErr(chiRouter, ns2, bktName1, objName2, apiErrors.ErrAccessDenied)
putObjectErr(chiRouter, ns2, bktName1, objName2, nil, apiErrors.ErrAccessDenied)
}
func TestPolicyCheckerReqTypeDetermination(t *testing.T) {
@ -215,6 +218,8 @@ func TestPolicyCheckerReqTypeDetermination(t *testing.T) {
_, _, err = chiRouter.policyChecker.MorphRuleChainStorage().AddMorphRuleChain(chain.S3, engine.NamespaceTarget(""), ruleChain)
require.NoError(t, err)
createBucket(chiRouter, "", bktName)
chiRouter.middlewareSettings.denyByDefault = true
t.Run("can list buckets", func(t *testing.T) {
w, r := httptest.NewRecorder(), httptest.NewRequest(http.MethodGet, "/", nil)
@ -263,9 +268,9 @@ func TestACLAPE(t *testing.T) {
router.middlewareSettings.denyByDefault = true
// Allow because of using old bucket
putObject(router, ns, bktNameOld, objName)
putObject(router, ns, bktNameOld, objName, nil)
// Deny because of deny by default
putObjectErr(router, ns, bktNameNew, objName, apiErrors.ErrAccessDenied)
putObjectErr(router, ns, bktNameNew, objName, nil, apiErrors.ErrAccessDenied)
// Deny because of deny by default
createBucketErr(router, ns, bktName, apiErrors.ErrAccessDenied)
@ -289,9 +294,9 @@ func TestACLAPE(t *testing.T) {
router.middlewareSettings.denyByDefault = false
// Allow because of using old bucket
putObject(router, ns, bktNameOld, objName)
putObject(router, ns, bktNameOld, objName, nil)
// Allow because of allow by default
putObject(router, ns, bktNameNew, objName)
putObject(router, ns, bktNameNew, objName, nil)
// Allow because of deny by default
createBucket(router, ns, bktName)
@ -315,9 +320,9 @@ func TestACLAPE(t *testing.T) {
router.middlewareSettings.denyByDefault = true
// Allow because of using old bucket
putObject(router, ns, bktNameOld, objName)
putObject(router, ns, bktNameOld, objName, nil)
// Deny because of deny by default
putObjectErr(router, ns, bktNameNew, objName, apiErrors.ErrAccessDenied)
putObjectErr(router, ns, bktNameNew, objName, nil, apiErrors.ErrAccessDenied)
// Allow because of old behavior
createBucket(router, ns, bktName)
@ -336,9 +341,9 @@ func TestACLAPE(t *testing.T) {
router.middlewareSettings.denyByDefault = false
// Allow because of using old bucket
putObject(router, ns, bktNameOld, objName)
putObject(router, ns, bktNameOld, objName, nil)
// Allow because of allow by default
putObject(router, ns, bktNameNew, objName)
putObject(router, ns, bktNameNew, objName, nil)
// Allow because of old behavior
createBucket(router, ns, bktName)
@ -459,6 +464,118 @@ func TestRequestParametersCheck(t *testing.T) {
})
}
func TestRequestTagsCheck(t *testing.T) {
t.Run("put bucket tagging", func(t *testing.T) {
router := prepareRouter(t)
ns, bktName, tagKey, tagValue := "", "bucket", "tag", "value"
router.middlewareSettings.denyByDefault = true
allowOperations(router, ns, []string{"s3:CreateBucket"}, nil)
createBucket(router, ns, bktName)
// Add policies and check
allowOperations(router, ns, []string{"s3:PutBucketTagging"}, engineiam.Conditions{
engineiam.CondStringEquals: engineiam.Condition{fmt.Sprintf(s3.PropertyKeyFormatRequestTag, tagKey): []string{tagValue}},
})
denyOperations(router, ns, []string{"s3:PutBucketTagging"}, engineiam.Conditions{
engineiam.CondStringNotEquals: engineiam.Condition{fmt.Sprintf(s3.PropertyKeyFormatRequestTag, tagKey): []string{tagValue}},
})
tagging, err := xml.Marshal(data.Tagging{TagSet: []data.Tag{{Key: tagKey, Value: tagValue}}})
require.NoError(t, err)
putBucketTagging(router, ns, bktName, tagging)
tagging, err = xml.Marshal(data.Tagging{TagSet: []data.Tag{{Key: "key", Value: tagValue}}})
require.NoError(t, err)
putBucketTaggingErr(router, ns, bktName, tagging, apiErrors.ErrAccessDenied)
})
t.Run("put object with tag", func(t *testing.T) {
router := prepareRouter(t)
ns, bktName, objName, tagKey, tagValue := "", "bucket", "object", "tag", "value"
router.middlewareSettings.denyByDefault = true
allowOperations(router, ns, []string{"s3:CreateBucket"}, nil)
createBucket(router, ns, bktName)
// Add policies and check
allowOperations(router, ns, []string{"s3:PutObject"}, engineiam.Conditions{
engineiam.CondStringEquals: engineiam.Condition{fmt.Sprintf(s3.PropertyKeyFormatRequestTag, tagKey): []string{tagValue}},
})
denyOperations(router, ns, []string{"s3:PutObject"}, engineiam.Conditions{
engineiam.CondStringNotEquals: engineiam.Condition{fmt.Sprintf(s3.PropertyKeyFormatRequestTag, tagKey): []string{tagValue}},
})
putObject(router, ns, bktName, objName, &data.Tag{Key: tagKey, Value: tagValue})
putObjectErr(router, ns, bktName, objName, &data.Tag{Key: "key", Value: tagValue}, apiErrors.ErrAccessDenied)
})
}
func TestResourceTagsCheck(t *testing.T) {
t.Run("bucket tagging", func(t *testing.T) {
router := prepareRouter(t)
ns, bktName, tagKey, tagValue := "", "bucket", "tag", "value"
router.middlewareSettings.denyByDefault = true
allowOperations(router, ns, []string{"s3:CreateBucket"}, nil)
createBucket(router, ns, bktName)
// Add policies and check
allowOperations(router, ns, []string{"s3:ListBucket"}, engineiam.Conditions{
engineiam.CondStringEquals: engineiam.Condition{fmt.Sprintf(s3.PropertyKeyFormatResourceTag, tagKey): []string{tagValue}},
})
denyOperations(router, ns, []string{"s3:ListBucket"}, engineiam.Conditions{
engineiam.CondStringNotEquals: engineiam.Condition{fmt.Sprintf(s3.PropertyKeyFormatResourceTag, tagKey): []string{tagValue}},
})
router.cfg.Tagging.(*resourceTaggingMock).bucketTags = map[string]string{tagKey: tagValue}
listObjectsV1(router, ns, bktName, "", "", "")
router.cfg.Tagging.(*resourceTaggingMock).bucketTags = map[string]string{}
listObjectsV1Err(router, ns, bktName, "", "", "", apiErrors.ErrAccessDenied)
})
t.Run("object tagging", func(t *testing.T) {
router := prepareRouter(t)
ns, bktName, objName, tagKey, tagValue := "", "bucket", "object", "tag", "value"
router.middlewareSettings.denyByDefault = true
allowOperations(router, ns, []string{"s3:CreateBucket", "s3:PutObject"}, nil)
createBucket(router, ns, bktName)
putObject(router, ns, bktName, objName, nil)
// Add policies and check
allowOperations(router, ns, []string{"s3:GetObject"}, engineiam.Conditions{
engineiam.CondStringEquals: engineiam.Condition{fmt.Sprintf(s3.PropertyKeyFormatResourceTag, tagKey): []string{tagValue}},
})
denyOperations(router, ns, []string{"s3:GetObject"}, engineiam.Conditions{
engineiam.CondStringNotEquals: engineiam.Condition{fmt.Sprintf(s3.PropertyKeyFormatResourceTag, tagKey): []string{tagValue}},
})
router.cfg.Tagging.(*resourceTaggingMock).objectTags = map[string]string{tagKey: tagValue}
getObject(router, ns, bktName, objName)
router.cfg.Tagging.(*resourceTaggingMock).objectTags = map[string]string{}
getObjectErr(router, ns, bktName, objName, apiErrors.ErrAccessDenied)
})
t.Run("non-existent resources", func(t *testing.T) {
router := prepareRouter(t)
ns, bktName, objName := "", "bucket", "object"
listObjectsV1Err(router, ns, bktName, "", "", "", apiErrors.ErrNoSuchBucket)
router.cfg.Tagging.(*resourceTaggingMock).noSuchKey = true
createBucket(router, ns, bktName)
getObjectErr(router, ns, bktName, objName, apiErrors.ErrNoSuchKey)
})
}
func allowOperations(router *routerMock, ns string, operations []string, conditions engineiam.Conditions) {
addPolicy(router, ns, "allow", engineiam.AllowEffect, operations, conditions)
}
@ -538,20 +655,68 @@ func listBucketsBase(router *routerMock, namespace string) *httptest.ResponseRec
return w
}
func putObject(router *routerMock, namespace, bktName, objName string) handlerResult {
w := putObjectBase(router, namespace, bktName, objName)
func putObject(router *routerMock, namespace, bktName, objName string, tag *data.Tag) handlerResult {
w := putObjectBase(router, namespace, bktName, objName, tag)
resp := readResponse(router.t, w)
require.Equal(router.t, s3middleware.PutObjectOperation, resp.Method)
return resp
}
func putObjectErr(router *routerMock, namespace, bktName, objName string, errCode apiErrors.ErrorCode) {
w := putObjectBase(router, namespace, bktName, objName)
func putObjectErr(router *routerMock, namespace, bktName, objName string, tag *data.Tag, errCode apiErrors.ErrorCode) {
w := putObjectBase(router, namespace, bktName, objName, tag)
assertAPIError(router.t, w, errCode)
}
func putObjectBase(router *routerMock, namespace, bktName, objName string) *httptest.ResponseRecorder {
func putObjectBase(router *routerMock, namespace, bktName, objName string, tag *data.Tag) *httptest.ResponseRecorder {
w, r := httptest.NewRecorder(), httptest.NewRequest(http.MethodPut, "/"+bktName+"/"+objName, nil)
if tag != nil {
queries := url.Values{
tag.Key: []string{tag.Value},
}
r.Header.Set(AmzTagging, queries.Encode())
}
r.Header.Set(FrostfsNamespaceHeader, namespace)
router.ServeHTTP(w, r)
return w
}
func putBucketTagging(router *routerMock, namespace, bktName string, tagging []byte) handlerResult {
w := putBucketTaggingBase(router, namespace, bktName, tagging)
resp := readResponse(router.t, w)
require.Equal(router.t, s3middleware.PutBucketTaggingOperation, resp.Method)
return resp
}
func putBucketTaggingErr(router *routerMock, namespace, bktName string, tagging []byte, errCode apiErrors.ErrorCode) {
w := putBucketTaggingBase(router, namespace, bktName, tagging)
assertAPIError(router.t, w, errCode)
}
func putBucketTaggingBase(router *routerMock, namespace, bktName string, tagging []byte) *httptest.ResponseRecorder {
queries := url.Values{}
queries.Add(s3middleware.TaggingQuery, "")
w, r := httptest.NewRecorder(), httptest.NewRequest(http.MethodPut, "/"+bktName, bytes.NewBuffer(tagging))
r.URL.RawQuery = queries.Encode()
r.Header.Set(FrostfsNamespaceHeader, namespace)
router.ServeHTTP(w, r)
return w
}
func getObject(router *routerMock, namespace, bktName, objName string) handlerResult {
w := getObjectBase(router, namespace, bktName, objName)
resp := readResponse(router.t, w)
require.Equal(router.t, s3middleware.GetObjectOperation, resp.Method)
return resp
}
func getObjectErr(router *routerMock, namespace, bktName, objName string, errCode apiErrors.ErrorCode) {
w := getObjectBase(router, namespace, bktName, objName)
assertAPIError(router.t, w, errCode)
}
func getObjectBase(router *routerMock, namespace, bktName, objName string) *httptest.ResponseRecorder {
w, r := httptest.NewRecorder(), httptest.NewRequest(http.MethodGet, "/"+bktName+"/"+objName, nil)
r.Header.Set(FrostfsNamespaceHeader, namespace)
router.ServeHTTP(w, r)
return w
@ -596,11 +761,11 @@ func TestOwnerIDRetrieving(t *testing.T) {
createBucket(chiRouter, ns, bktName)
resp := putObject(chiRouter, ns, bktName, objName)
resp := putObject(chiRouter, ns, bktName, objName, nil)
require.NotEqual(t, "anon", resp.ReqInfo.User)
chiRouter.cfg.Center.(*centerMock).anon = true
resp = putObject(chiRouter, ns, bktName, objName)
resp = putObject(chiRouter, ns, bktName, objName, nil)
require.Equal(t, "anon", resp.ReqInfo.User)
}
@ -618,7 +783,7 @@ func TestBillingMetrics(t *testing.T) {
require.Equal(t, 1, dump.Requests[0].Requests)
chiRouter.cfg.Center.(*centerMock).anon = true
putObject(chiRouter, ns, bktName, objName)
putObject(chiRouter, ns, bktName, objName, nil)
dump = chiRouter.cfg.Metrics.UsersAPIStats().DumpMetrics()
require.Len(t, dump.Requests, 1)
require.Equal(t, "anon", dump.Requests[0].User)

3
cmd/s3-gw/app.go
View file

@ -688,6 +688,9 @@ func (a *App) Serve(ctx context.Context) {
FrostfsID: a.frostfsid,
FrostFSIDValidation: a.settings.frostfsidValidation,
XMLDecoder: a.settings,
Tagging: a.obj,
}
chiRouter := api.NewRouter(cfg)