bugfix/306-use_APE_instead_eACL #310

Merged
alexvanin merged 15 commits from dkirillov/frostfs-s3-gw:bugfix/306-use_APE_instead_eACL into master 2024-09-04 19:51:13 +00:00
36 changed files with 1381 additions and 583 deletions

View file

@ -32,7 +32,8 @@ This document outlines major changes between releases.
- Set server IdleTimeout and ReadHeaderTimeout to `30s` and allow to configure them (#220)
- Return `ETag` value in quotes (#219)
- Use tombstone when delete multipart upload (#275)
- Support new parameter `cache.accessbox.removing_check_interval` (#XX)
- Support new parameter `cache.accessbox.removing_check_interval` (#305)
- Use APE rules instead of eACL in container creation (#306)
### Removed
- Drop sending whitespace characters during complete multipart upload and related config param `kludge.complete_multipart_keepalive` (#227)

View file

@ -8,6 +8,7 @@ import (
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user"
"github.com/nspcc-dev/neo-go/pkg/crypto/keys"
)
const (
@ -31,6 +32,7 @@ type (
LocationConstraint string
ObjectLockEnabled bool
HomomorphicHashDisabled bool
APEEnabled bool
}
// ObjectInfo holds S3 object data.
@ -60,8 +62,10 @@ type (
// BucketSettings stores settings such as versioning.
BucketSettings struct {
Versioning string `json:"versioning"`
LockConfiguration *ObjectLockConfiguration `json:"lock_configuration"`
Versioning string
LockConfiguration *ObjectLockConfiguration
CannedACL string
OwnerKey *keys.PublicKey
}
// CORSConfiguration stores CORS configuration of a request.

View file

@ -26,6 +26,7 @@ type (
const (
_ ErrorCode = iota
ErrAccessDenied
ErrAccessControlListNotSupported
ErrBadDigest
ErrEntityTooSmall
ErrEntityTooLarge
@ -376,6 +377,12 @@ var errorCodes = errorCodeMap{
Description: "Access Denied.",
HTTPStatusCode: http.StatusForbidden,
},
ErrAccessControlListNotSupported: {
ErrCode: ErrAccessControlListNotSupported,
Code: "AccessControlListNotSupported",
Description: "The bucket does not allow ACLs.",
HTTPStatusCode: http.StatusBadRequest,
},
ErrBadDigest: {
ErrCode: ErrBadDigest,
Code: "BadDigest",

View file

@ -20,6 +20,7 @@ import (
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/middleware"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/creds/accessbox"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/eacl"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
@ -27,7 +28,6 @@ import (
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/session"
engineiam "git.frostfs.info/TrueCloudLab/policy-engine/iam"
"git.frostfs.info/TrueCloudLab/policy-engine/pkg/chain"
"git.frostfs.info/TrueCloudLab/policy-engine/pkg/engine"
"github.com/nspcc-dev/neo-go/pkg/crypto/keys"
"go.uber.org/zap"
)
@ -257,6 +257,20 @@ func (h *handler) GetBucketACLHandler(w http.ResponseWriter, r *http.Request) {
return
}
settings, err := h.obj.GetBucketSettings(r.Context(), bktInfo)
if err != nil {
h.logAndSendError(w, "couldn't get bucket settings", reqInfo, err)
return
}
if bktInfo.APEEnabled || len(settings.CannedACL) != 0 {
if err = middleware.EncodeToResponse(w, h.encodeBucketCannedACL(ctx, bktInfo, settings)); err != nil {
h.logAndSendError(w, "something went wrong", reqInfo, err)
return
}
return
}
bucketACL, err := h.obj.GetBucketACL(ctx, bktInfo)
if err != nil {
h.logAndSendError(w, "could not fetch bucket acl", reqInfo, err)
@ -269,16 +283,69 @@ func (h *handler) GetBucketACLHandler(w http.ResponseWriter, r *http.Request) {
}
}
func (h *handler) encodeBucketCannedACL(ctx context.Context, bktInfo *data.BucketInfo, settings *data.BucketSettings) *AccessControlPolicy {
ownerDisplayName := bktInfo.Owner.EncodeToString()
ownerEncodedID := ownerDisplayName
if settings.OwnerKey == nil {
h.reqLogger(ctx).Warn(logs.BucketOwnerKeyIsMissing, zap.String("owner", bktInfo.Owner.String()))
} else {
ownerDisplayName = settings.OwnerKey.Address()
ownerEncodedID = hex.EncodeToString(settings.OwnerKey.Bytes())
}
res := &AccessControlPolicy{Owner: Owner{
ID: ownerEncodedID,
DisplayName: ownerDisplayName,
}}
granteeOwner := NewGrantee(acpCanonicalUser)
granteeOwner.ID = ownerEncodedID
granteeOwner.DisplayName = ownerDisplayName
res.AccessControlList = []*Grant{{
Grantee: granteeOwner,
Permission: aclFullControl,
}}
switch settings.CannedACL {
case basicACLPublic:
grantee := NewGrantee(acpGroup)
grantee.URI = allUsersGroup
res.AccessControlList = append(res.AccessControlList, &Grant{
Grantee: grantee,
Permission: aclWrite,
})
fallthrough
case basicACLReadOnly:
grantee := NewGrantee(acpGroup)
grantee.URI = allUsersGroup
res.AccessControlList = append(res.AccessControlList, &Grant{
Grantee: grantee,
Permission: aclRead,
})
}
return res
}
func (h *handler) bearerTokenIssuerKey(ctx context.Context) (*keys.PublicKey, error) {
box, err := middleware.GetBoxData(ctx)
if err != nil {
return nil, err
}
var btoken v2acl.BearerToken
box.Gate.BearerToken.WriteToV2(&btoken)
return getTokenIssuerKey(box)
}
key, err := keys.NewPublicKeyFromBytes(btoken.GetSignature().GetKey(), elliptic.P256())
func getTokenIssuerKey(box *accessbox.Box) (*keys.PublicKey, error) {
if box.Gate.BearerToken == nil {
return nil, stderrors.New("bearer token is missing")
}
key, err := keys.NewPublicKeyFromBytes(box.Gate.BearerToken.SigningKeyBytes(), elliptic.P256())
if err != nil {
return nil, fmt.Errorf("public key from bytes: %w", err)
}
@ -288,6 +355,24 @@ func (h *handler) bearerTokenIssuerKey(ctx context.Context) (*keys.PublicKey, er
func (h *handler) PutBucketACLHandler(w http.ResponseWriter, r *http.Request) {
reqInfo := middleware.GetReqInfo(r.Context())
bktInfo, err := h.getBucketAndCheckOwner(r, reqInfo.BucketName)
if err != nil {
h.logAndSendError(w, "could not get bucket info", reqInfo, err)
return
}
settings, err := h.obj.GetBucketSettings(r.Context(), bktInfo)
if err != nil {
h.logAndSendError(w, "couldn't get bucket settings", reqInfo, err)
return
}
if bktInfo.APEEnabled || len(settings.CannedACL) != 0 {
h.putBucketACLAPEHandler(w, r, reqInfo, bktInfo, settings)
return
}
key, err := h.bearerTokenIssuerKey(r.Context())
if err != nil {
h.logAndSendError(w, "couldn't get bearer token issuer key", reqInfo, err)
@ -319,12 +404,6 @@ func (h *handler) PutBucketACLHandler(w http.ResponseWriter, r *http.Request) {
return
}
bktInfo, err := h.getBucketAndCheckOwner(r, reqInfo.BucketName)
if err != nil {
h.logAndSendError(w, "could not get bucket info", reqInfo, err)
return
}
if _, err = h.updateBucketACL(r, astBucket, bktInfo, token); err != nil {
h.logAndSendError(w, "could not update bucket acl", reqInfo, err)
return
@ -332,6 +411,60 @@ func (h *handler) PutBucketACLHandler(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusOK)
}
func (h *handler) putBucketACLAPEHandler(w http.ResponseWriter, r *http.Request, reqInfo *middleware.ReqInfo, bktInfo *data.BucketInfo, settings *data.BucketSettings) {
ctx := r.Context()
defer func() {
if errBody := r.Body.Close(); errBody != nil {
h.reqLogger(r.Context()).Warn(logs.CouldNotCloseRequestBody, zap.Error(errBody))
}
}()
written, err := io.Copy(io.Discard, r.Body)
if err != nil {
h.logAndSendError(w, "couldn't read request body", reqInfo, err)
return
}
if written != 0 || len(r.Header.Get(api.AmzACL)) == 0 {
h.logAndSendError(w, "acl not supported for this bucket", reqInfo, errors.GetAPIError(errors.ErrAccessControlListNotSupported))
return
}
cannedACL, err := parseCannedACL(r.Header)
if err != nil {
h.logAndSendError(w, "could not parse canned ACL", reqInfo, err)
return
}
key, err := h.bearerTokenIssuerKey(ctx)
if err != nil {
h.logAndSendError(w, "couldn't get bearer token issuer key", reqInfo, err)
return
}
chainRules := bucketCannedACLToAPERules(cannedACL, reqInfo, key, bktInfo.CID)
if err = h.ape.SaveACLChains(reqInfo.Namespace, chainRules); err != nil {
h.logAndSendError(w, "failed to add morph rule chains", reqInfo, err)
return
}
settings.CannedACL = cannedACL
sp := &layer.PutSettingsParams{
BktInfo: bktInfo,
Settings: settings,
}
if err = h.obj.PutBucketSettings(ctx, sp); err != nil {
h.logAndSendError(w, "couldn't save bucket settings", reqInfo, err,
zap.String("container_id", bktInfo.CID.EncodeToString()))
return
}
w.WriteHeader(http.StatusOK)
}
func (h *handler) updateBucketACL(r *http.Request, astChild *ast, bktInfo *data.BucketInfo, sessionToken *session.Container) (bool, error) {
bucketACL, err := h.obj.GetBucketACL(r.Context(), bktInfo)
if err != nil {
@ -380,6 +513,22 @@ func (h *handler) GetObjectACLHandler(w http.ResponseWriter, r *http.Request) {
return
}
apeEnabled := bktInfo.APEEnabled
if !apeEnabled {
settings, err := h.obj.GetBucketSettings(r.Context(), bktInfo)
if err != nil {
h.logAndSendError(w, "couldn't get bucket settings", reqInfo, err)
return
}
apeEnabled = len(settings.CannedACL) != 0
}
if apeEnabled {
h.logAndSendError(w, "acl not supported for this bucket", reqInfo, errors.GetAPIError(errors.ErrAccessControlListNotSupported))
return
}
bucketACL, err := h.obj.GetBucketACL(ctx, bktInfo)
if err != nil {
h.logAndSendError(w, "could not fetch bucket acl", reqInfo, err)
@ -406,6 +555,29 @@ func (h *handler) GetObjectACLHandler(w http.ResponseWriter, r *http.Request) {
func (h *handler) PutObjectACLHandler(w http.ResponseWriter, r *http.Request) {
ctx := r.Context()
reqInfo := middleware.GetReqInfo(ctx)
bktInfo, err := h.getBucketAndCheckOwner(r, reqInfo.BucketName)
if err != nil {
h.logAndSendError(w, "could not get bucket info", reqInfo, err)
return
}
apeEnabled := bktInfo.APEEnabled
if !apeEnabled {
settings, err := h.obj.GetBucketSettings(r.Context(), bktInfo)
if err != nil {
h.logAndSendError(w, "couldn't get bucket settings", reqInfo, err)
return
}
apeEnabled = len(settings.CannedACL) != 0
}
if apeEnabled {
h.logAndSendError(w, "acl not supported for this bucket", reqInfo, errors.GetAPIError(errors.ErrAccessControlListNotSupported))
return
}
versionID := reqInfo.URL.Query().Get(api.QueryVersionID)
key, err := h.bearerTokenIssuerKey(ctx)
if err != nil {
@ -419,12 +591,6 @@ func (h *handler) PutObjectACLHandler(w http.ResponseWriter, r *http.Request) {
return
}
bktInfo, err := h.getBucketAndCheckOwner(r, reqInfo.BucketName)
if err != nil {
h.logAndSendError(w, "could not get bucket info", reqInfo, err)
return
}
p := &layer.HeadObjectParams{
BktInfo: bktInfo,
Object: reqInfo.ObjectName,
@ -489,8 +655,7 @@ func (h *handler) GetBucketPolicyHandler(w http.ResponseWriter, r *http.Request)
return
}
resolvedNamespace := h.cfg.ResolveNamespaceAlias(reqInfo.Namespace)
jsonPolicy, err := h.ape.GetPolicy(resolvedNamespace, bktInfo.CID)
jsonPolicy, err := h.ape.GetBucketPolicy(reqInfo.Namespace, bktInfo.CID)
if err != nil {
if strings.Contains(err.Error(), "not found") {
err = fmt.Errorf("%w: %s", errors.GetAPIError(errors.ErrNoSuchBucketPolicy), err.Error())
@ -516,16 +681,7 @@ func (h *handler) DeleteBucketPolicyHandler(w http.ResponseWriter, r *http.Reque
return
}
resolvedNamespace := h.cfg.ResolveNamespaceAlias(reqInfo.Namespace)
target := engine.NamespaceTarget(resolvedNamespace)
chainID := getBucketChainID(bktInfo)
if err = h.ape.RemoveChain(target, chainID); err != nil {
h.logAndSendError(w, "failed to remove morph rule chain", reqInfo, err)
return
}
if err = h.ape.DeletePolicy(resolvedNamespace, bktInfo.CID); err != nil {
if err = h.ape.DeleteBucketPolicy(reqInfo.Namespace, bktInfo.CID, getBucketChainID(chain.S3, bktInfo)); err != nil {
h.logAndSendError(w, "failed to delete policy from storage", reqInfo, err)
return
}
@ -565,15 +721,13 @@ func (h *handler) PutBucketPolicyHandler(w http.ResponseWriter, r *http.Request)
return
}
s3Chain, err := engineiam.ConvertToS3Chain(bktPolicy, h.frostfsid)
if err != nil {
h.logAndSendError(w, "could not convert s3 policy to chain policy", reqInfo, err)
return
}
s3Chain.ID = getBucketChainID(bktInfo)
for _, stat := range bktPolicy.Statement {
if len(stat.NotResource) != 0 {
h.logAndSendError(w, "policy resource mismatched bucket", reqInfo, errors.GetAPIError(errors.ErrMalformedPolicy))
return
}
for _, rule := range s3Chain.Rules {
for _, resource := range rule.Resources.Names {
for _, resource := range stat.Resource {
if reqInfo.BucketName != strings.Split(strings.TrimPrefix(resource, arnAwsPrefix), "/")[0] {
h.logAndSendError(w, "policy resource mismatched bucket", reqInfo, errors.GetAPIError(errors.ErrMalformedPolicy))
return
@ -581,22 +735,50 @@ func (h *handler) PutBucketPolicyHandler(w http.ResponseWriter, r *http.Request)
}
}
resolvedNamespace := h.cfg.ResolveNamespaceAlias(reqInfo.Namespace)
target := engine.NamespaceTarget(resolvedNamespace)
if err = h.ape.AddChain(target, s3Chain); err != nil {
h.logAndSendError(w, "failed to add morph rule chain", reqInfo, err)
nativeChain, err := engineiam.ConvertToNativeChain(bktPolicy, h.nativeResolver(reqInfo.Namespace, bktInfo))
if err != nil {
h.logAndSendError(w, "could not convert s3 policy to native chain policy", reqInfo, err)
return
}
nativeChain.ID = getBucketChainID(chain.Ingress, bktInfo)
if err = h.ape.PutPolicy(resolvedNamespace, bktInfo.CID, jsonPolicy); err != nil {
h.logAndSendError(w, "failed to save policy to storage", reqInfo, err)
s3Chain, err := engineiam.ConvertToS3Chain(bktPolicy, h.frostfsid)
if err != nil {
h.logAndSendError(w, "could not convert s3 policy to chain policy", reqInfo, err)
return
}
s3Chain.ID = getBucketChainID(chain.S3, bktInfo)
if err = h.ape.PutBucketPolicy(reqInfo.Namespace, bktInfo.CID, jsonPolicy, []*chain.Chain{s3Chain, nativeChain}); err != nil {
h.logAndSendError(w, "failed to update policy in contract", reqInfo, err)
return
}
}
func getBucketChainID(bktInfo *data.BucketInfo) chain.ID {
return chain.ID("bkt" + string(bktInfo.CID[:]))
type nativeResolver struct {
FrostFSID
namespace string
bktInfo *data.BucketInfo
}
func (n *nativeResolver) GetBucketInfo(bucket string) (*engineiam.BucketInfo, error) {
if n.bktInfo.Name != bucket {
return nil, fmt.Errorf("invalid bucket %s: %w", bucket, errors.GetAPIError(errors.ErrMalformedPolicy))
}
return &engineiam.BucketInfo{Namespace: n.namespace, Container: n.bktInfo.CID.EncodeToString()}, nil
}
func (h *handler) nativeResolver(ns string, bktInfo *data.BucketInfo) engineiam.NativeResolver {
return &nativeResolver{
FrostFSID: h.frostfsid,
namespace: ns,
bktInfo: bktInfo,
}
}
func getBucketChainID(prefix chain.Name, bktInfo *data.BucketInfo) chain.ID {
return chain.ID(string(prefix) + ":bkt" + string(bktInfo.CID[:]))
}
func parseACLHeaders(header http.Header, key *keys.PublicKey) (*AccessControlPolicy, error) {

View file

@ -24,6 +24,7 @@ import (
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/session"
engineiam "git.frostfs.info/TrueCloudLab/policy-engine/iam"
"git.frostfs.info/TrueCloudLab/policy-engine/pkg/engine"
"github.com/nspcc-dev/neo-go/pkg/crypto/keys"
"github.com/stretchr/testify/require"
)
@ -1300,17 +1301,102 @@ func TestBucketAclToAst(t *testing.T) {
func TestPutBucketACL(t *testing.T) {
tc := prepareHandlerContext(t)
tc.config.aclEnabled = true
bktName := "bucket-for-acl"
box, _ := createAccessBox(t)
bktInfo := createBucket(t, tc, bktName, box)
info := createBucket(tc, bktName)
header := map[string]string{api.AmzACL: "public-read"}
putBucketACL(t, tc, bktName, box, header)
putBucketACL(tc, bktName, info.Box, header)
header = map[string]string{api.AmzACL: "private"}
putBucketACL(t, tc, bktName, box, header)
checkLastRecords(t, tc, bktInfo, eacl.ActionDeny)
putBucketACL(tc, bktName, info.Box, header)
checkLastRecords(t, tc, info.BktInfo, eacl.ActionDeny)
}
func TestPutBucketAPE(t *testing.T) {
hc := prepareHandlerContext(t)
bktName := "bucket-for-acl-ape"
info := createBucket(hc, bktName)
_, err := hc.tp.ContainerEACL(hc.Context(), info.BktInfo.CID)
require.ErrorContains(t, err, "not found")
chains, err := hc.h.ape.(*apeMock).ListChains(engine.NamespaceTarget(""))
require.NoError(t, err)
require.Len(t, chains, 2)
}
func TestPutBucketObjectACLErrorAPE(t *testing.T) {
hc := prepareHandlerContext(t)
bktName, objName := "bucket-for-acl-ape", "object"
info := createBucket(hc, bktName)
putObject(hc, bktName, objName)
aclBody := &AccessControlPolicy{}
putBucketACLAssertS3Error(hc, bktName, info.Box, nil, aclBody, s3errors.ErrAccessControlListNotSupported)
putObjectACLAssertS3Error(hc, bktName, objName, info.Box, nil, aclBody, s3errors.ErrAccessControlListNotSupported)
getObjectACLAssertS3Error(hc, bktName, objName, s3errors.ErrAccessControlListNotSupported)
}
func TestGetBucketACLAPE(t *testing.T) {
hc := prepareHandlerContext(t)
bktName := "bucket-for-acl-ape"
info := createBucket(hc, bktName)
aclRes := getBucketACL(hc, bktName)
checkPrivateBucketACL(t, aclRes, info.Key.PublicKey())
putBucketACL(hc, bktName, info.Box, map[string]string{api.AmzACL: basicACLPrivate})
aclRes = getBucketACL(hc, bktName)
checkPrivateBucketACL(t, aclRes, info.Key.PublicKey())
putBucketACL(hc, bktName, info.Box, map[string]string{api.AmzACL: basicACLReadOnly})
aclRes = getBucketACL(hc, bktName)
checkPublicReadBucketACL(t, aclRes, info.Key.PublicKey())
putBucketACL(hc, bktName, info.Box, map[string]string{api.AmzACL: basicACLPublic})
aclRes = getBucketACL(hc, bktName)
checkPublicReadWriteBucketACL(t, aclRes, info.Key.PublicKey())
}
func checkPrivateBucketACL(t *testing.T, aclRes *AccessControlPolicy, ownerKey *keys.PublicKey) {
checkBucketACLOwner(t, aclRes, ownerKey, 1)
}
func checkPublicReadBucketACL(t *testing.T, aclRes *AccessControlPolicy, ownerKey *keys.PublicKey) {
checkBucketACLOwner(t, aclRes, ownerKey, 2)
require.Equal(t, allUsersGroup, aclRes.AccessControlList[1].Grantee.URI)
require.Equal(t, aclRead, aclRes.AccessControlList[1].Permission)
}
func checkPublicReadWriteBucketACL(t *testing.T, aclRes *AccessControlPolicy, ownerKey *keys.PublicKey) {
checkBucketACLOwner(t, aclRes, ownerKey, 3)
require.Equal(t, allUsersGroup, aclRes.AccessControlList[1].Grantee.URI)
require.Equal(t, aclWrite, aclRes.AccessControlList[1].Permission)
require.Equal(t, allUsersGroup, aclRes.AccessControlList[2].Grantee.URI)
require.Equal(t, aclRead, aclRes.AccessControlList[2].Permission)
}
func checkBucketACLOwner(t *testing.T, aclRes *AccessControlPolicy, ownerKey *keys.PublicKey, ln int) {
ownerIDStr := hex.EncodeToString(ownerKey.Bytes())
ownerNameStr := ownerKey.Address()
require.Equal(t, ownerIDStr, aclRes.Owner.ID)
require.Equal(t, ownerNameStr, aclRes.Owner.DisplayName)
require.Len(t, aclRes.AccessControlList, ln)
require.Equal(t, ownerIDStr, aclRes.AccessControlList[0].Grantee.ID)
require.Equal(t, ownerNameStr, aclRes.AccessControlList[0].Grantee.DisplayName)
require.Equal(t, aclFullControl, aclRes.AccessControlList[0].Permission)
}
func TestBucketPolicy(t *testing.T) {
@ -1488,13 +1574,26 @@ func createAccessBox(t *testing.T) (*accessbox.Box, *keys.PrivateKey) {
return box, key
}
func createBucket(t *testing.T, hc *handlerContext, bktName string, box *accessbox.Box) *data.BucketInfo {
type createBucketInfo struct {
BktInfo *data.BucketInfo
Box *accessbox.Box
Key *keys.PrivateKey
}
func createBucket(hc *handlerContext, bktName string) *createBucketInfo {
box, key := createAccessBox(hc.t)
w := createBucketBase(hc, bktName, box)
assertStatus(t, w, http.StatusOK)
assertStatus(hc.t, w, http.StatusOK)
bktInfo, err := hc.Layer().GetBucketInfo(hc.Context(), bktName)
require.NoError(t, err)
return bktInfo
require.NoError(hc.t, err)
return &createBucketInfo{
BktInfo: bktInfo,
Box: box,
Key: key,
}
}
func createBucketAssertS3Error(hc *handlerContext, bktName string, box *accessbox.Box, code s3errors.ErrorCode) {
@ -1510,13 +1609,64 @@ func createBucketBase(hc *handlerContext, bktName string, box *accessbox.Box) *h
return w
}
func putBucketACL(t *testing.T, tc *handlerContext, bktName string, box *accessbox.Box, header map[string]string) {
w, r := prepareTestRequest(tc, bktName, "", nil)
func putBucketACL(hc *handlerContext, bktName string, box *accessbox.Box, header map[string]string) {
w := putBucketACLBase(hc, bktName, box, header, nil)
assertStatus(hc.t, w, http.StatusOK)
}
func putBucketACLAssertS3Error(hc *handlerContext, bktName string, box *accessbox.Box, header map[string]string, body *AccessControlPolicy, code s3errors.ErrorCode) {
w := putBucketACLBase(hc, bktName, box, header, body)
assertS3Error(hc.t, w, s3errors.GetAPIError(code))
}
func putBucketACLBase(hc *handlerContext, bktName string, box *accessbox.Box, header map[string]string, body *AccessControlPolicy) *httptest.ResponseRecorder {
w, r := prepareTestRequest(hc, bktName, "", body)
for key, val := range header {
r.Header.Set(key, val)
}
ctx := middleware.SetBoxData(r.Context(), box)
r = r.WithContext(ctx)
tc.Handler().PutBucketACLHandler(w, r)
assertStatus(t, w, http.StatusOK)
hc.Handler().PutBucketACLHandler(w, r)
return w
}
func getBucketACL(hc *handlerContext, bktName string) *AccessControlPolicy {
w := getBucketACLBase(hc, bktName)
assertStatus(hc.t, w, http.StatusOK)
res := &AccessControlPolicy{}
parseTestResponse(hc.t, w, res)
return res
}
func getBucketACLBase(hc *handlerContext, bktName string) *httptest.ResponseRecorder {
w, r := prepareTestRequest(hc, bktName, "", nil)
hc.Handler().GetBucketACLHandler(w, r)
return w
}
func putObjectACLAssertS3Error(hc *handlerContext, bktName, objName string, box *accessbox.Box, header map[string]string, body *AccessControlPolicy, code s3errors.ErrorCode) {
w := putObjectACLBase(hc, bktName, objName, box, header, body)
assertS3Error(hc.t, w, s3errors.GetAPIError(code))
}
func putObjectACLBase(hc *handlerContext, bktName, objName string, box *accessbox.Box, header map[string]string, body *AccessControlPolicy) *httptest.ResponseRecorder {
w, r := prepareTestRequest(hc, bktName, objName, body)
for key, val := range header {
r.Header.Set(key, val)
}
ctx := middleware.SetBoxData(r.Context(), box)
r = r.WithContext(ctx)
hc.Handler().PutObjectACLHandler(w, r)
return w
}
func getObjectACLAssertS3Error(hc *handlerContext, bktName, objName string, code s3errors.ErrorCode) {
w := getObjectACLBase(hc, bktName, objName)
assertS3Error(hc.t, w, s3errors.GetAPIError(code))
}
func getObjectACLBase(hc *handlerContext, bktName, objName string) *httptest.ResponseRecorder {
w, r := prepareTestRequest(hc, bktName, objName, nil)
hc.Handler().GetObjectACLHandler(w, r)
return w
}

View file

@ -15,7 +15,6 @@ import (
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap"
"git.frostfs.info/TrueCloudLab/policy-engine/pkg/chain"
"git.frostfs.info/TrueCloudLab/policy-engine/pkg/engine"
"go.uber.org/zap"
)
@ -47,35 +46,21 @@ type (
IsResolveListAllow() bool
BypassContentEncodingInChunks() bool
MD5Enabled() bool
ResolveNamespaceAlias(namespace string) string
ACLEnabled() bool
}
FrostFSID interface {
GetUserAddress(account, user string) (string, error)
GetUserKey(account, name string) (string, error)
}
// APE is Access Policy Engine that needs to save policy and acl info to different places.
APE interface {
MorphRuleChainStorage
PolicyStorage
PutBucketPolicy(ns string, cnrID cid.ID, policy []byte, chains []*chain.Chain) error
DeleteBucketPolicy(ns string, cnrID cid.ID, chainID chain.ID) error
GetBucketPolicy(ns string, cnrID cid.ID) ([]byte, error)
SaveACLChains(ns string, chains []*chain.Chain) error
}
// MorphRuleChainStorage is a similar to engine.MorphRuleChainStorage
// but doesn't know anything about tx.
MorphRuleChainStorage interface {
AddChain(target engine.Target, c *chain.Chain) error
RemoveChain(target engine.Target, chainID chain.ID) error
ListChains(target engine.Target) ([]*chain.Chain, error)
}
// PolicyStorage is interface to save intact initial user provided policy.
PolicyStorage interface {
PutPolicy(namespace string, cnrID cid.ID, policy []byte) error
GetPolicy(namespace string, cnrID cid.ID) ([]byte, error)
DeletePolicy(namespace string, cnrID cid.ID) error
}
frostfsIDDisabled struct{}
)
var _ api.Handler = (*handler)(nil)
@ -89,10 +74,8 @@ func New(log *zap.Logger, obj layer.Client, notificator Notificator, cfg Config,
return nil, errors.New("empty logger")
case storage == nil:
return nil, errors.New("empty policy storage")
}
if ffsid == nil {
ffsid = frostfsIDDisabled{}
case ffsid == nil:
return nil, errors.New("empty frostfsid")
}
if !cfg.NotificatorEnabled() {
@ -111,10 +94,6 @@ func New(log *zap.Logger, obj layer.Client, notificator Notificator, cfg Config,
}, nil
}
func (f frostfsIDDisabled) GetUserAddress(_, _ string) (string, error) {
return "", errors.New("frostfsid disabled")
}
// pickCopiesNumbers chooses the return values following this logic:
// 1) array of copies numbers sent in request's header has the highest priority.
// 2) array of copies numbers with corresponding location constraint provided in the config file.

View file

@ -168,7 +168,7 @@ func TestDeleteDeletedObject(t *testing.T) {
})
t.Run("versioned bucket not found obj", func(t *testing.T) {
bktName, objName := "bucket-versioned-for-removal", "object-to-delete"
bktName, objName := "bucket-versioned-for-removal-not-found", "object-to-delete"
_, objInfo := createVersionedBucketAndObject(t, tc, bktName, objName)
versionID, isDeleteMarker := deleteObject(t, tc, bktName, objName, objInfo.VersionID())

View file

@ -21,7 +21,6 @@ import (
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/middleware"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/resolver"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/pkg/service/tree"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/acl"
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
@ -71,6 +70,7 @@ type configMock struct {
defaultCopiesNumbers []uint32
bypassContentEncodingInChunks bool
md5Enabled bool
aclEnabled bool
}
func (c *configMock) DefaultPlacementPolicy(_ string) netmap.PlacementPolicy {
@ -122,6 +122,10 @@ func (c *configMock) MD5Enabled() bool {
return c.md5Enabled
}
func (c *configMock) ACLEnabled() bool {
return c.aclEnabled
}
func (c *configMock) ResolveNamespaceAlias(ns string) string {
return ns
}
@ -252,8 +256,34 @@ func (a *apeMock) PutPolicy(namespace string, cnrID cid.ID, policy []byte) error
return nil
}
func (a *apeMock) GetPolicy(namespace string, cnrID cid.ID) ([]byte, error) {
policy, ok := a.policyMap[namespace+cnrID.EncodeToString()]
func (a *apeMock) DeletePolicy(namespace string, cnrID cid.ID) error {
delete(a.policyMap, namespace+cnrID.EncodeToString())
return nil
}
func (a *apeMock) PutBucketPolicy(ns string, cnrID cid.ID, policy []byte, chain []*chain.Chain) error {
if err := a.PutPolicy(ns, cnrID, policy); err != nil {
return err
}
for i := range chain {
if err := a.AddChain(engine.NamespaceTarget(ns), chain[i]); err != nil {
return err
}
}
return nil
}
func (a *apeMock) DeleteBucketPolicy(ns string, cnrID cid.ID, chainID chain.ID) error {
if err := a.DeletePolicy(ns, cnrID); err != nil {
return err
}
return a.RemoveChain(engine.NamespaceTarget(ns), chainID)
}
func (a *apeMock) GetBucketPolicy(ns string, cnrID cid.ID) ([]byte, error) {
policy, ok := a.policyMap[ns+cnrID.EncodeToString()]
if !ok {
return nil, errors.New("not found")
}
@ -261,22 +291,19 @@ func (a *apeMock) GetPolicy(namespace string, cnrID cid.ID) ([]byte, error) {
return policy, nil
}
func (a *apeMock) DeletePolicy(namespace string, cnrID cid.ID) error {
delete(a.policyMap, namespace+cnrID.EncodeToString())
func (a *apeMock) SaveACLChains(ns string, chains []*chain.Chain) error {
for i := range chains {
if err := a.AddChain(engine.NamespaceTarget(ns), chains[i]); err != nil {
return err
}
}
return nil
}
func createTestBucket(hc *handlerContext, bktName string) *data.BucketInfo {
_, err := hc.MockedPool().CreateContainer(hc.Context(), layer.PrmContainerCreate{
Creator: hc.owner,
Name: bktName,
BasicACL: acl.PublicRWExtended,
})
require.NoError(hc.t, err)
bktInfo, err := hc.Layer().GetBucketInfo(hc.Context(), bktName)
require.NoError(hc.t, err)
return bktInfo
info := createBucket(hc, bktName)
return info.BktInfo
}
func createTestBucketWithLock(hc *handlerContext, bktName string, conf *data.ObjectLockConfiguration) *data.BucketInfo {
@ -297,11 +324,15 @@ func createTestBucketWithLock(hc *handlerContext, bktName string, conf *data.Obj
HomomorphicHashDisabled: res.HomomorphicHashDisabled,
}
key, err := keys.NewPrivateKey()
require.NoError(hc.t, err)
sp := &layer.PutSettingsParams{
BktInfo: bktInfo,
Settings: &data.BucketSettings{
Versioning: data.VersioningEnabled,
LockConfiguration: conf,
OwnerKey: key.PublicKey(),
},
}

View file

@ -4,6 +4,7 @@ import (
"bytes"
"crypto/md5"
"encoding/base64"
"encoding/hex"
"encoding/json"
"encoding/xml"
"fmt"
@ -24,8 +25,13 @@ import (
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/middleware"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/creds/accessbox"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs"
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/eacl"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/session"
"git.frostfs.info/TrueCloudLab/policy-engine/pkg/chain"
"git.frostfs.info/TrueCloudLab/policy-engine/schema/native"
"git.frostfs.info/TrueCloudLab/policy-engine/schema/s3"
"github.com/nspcc-dev/neo-go/pkg/crypto/keys"
"go.uber.org/zap"
)
@ -744,22 +750,140 @@ func parseMetadata(r *http.Request) map[string]string {
return res
}
func (h *handler) CreateBucketHandler(w http.ResponseWriter, r *http.Request) {
ctx := r.Context()
reqInfo := middleware.GetReqInfo(ctx)
p := &layer.CreateBucketParams{
Name: reqInfo.BucketName,
Namespace: reqInfo.Namespace,
func parseCannedACL(header http.Header) (string, error) {
acl := header.Get(api.AmzACL)
if len(acl) == 0 {
return basicACLPrivate, nil
}
if err := checkBucketName(reqInfo.BucketName); err != nil {
h.logAndSendError(w, "invalid bucket name", reqInfo, err)
if acl == basicACLPrivate || acl == basicACLPublic ||
acl == cannedACLAuthRead || acl == basicACLReadOnly {
return acl, nil
}
return "", fmt.Errorf("unknown acl: %s", acl)
}
func (h *handler) CreateBucketHandler(w http.ResponseWriter, r *http.Request) {
if h.cfg.ACLEnabled() {
h.createBucketHandlerACL(w, r)
return
}
key, err := h.bearerTokenIssuerKey(ctx)
h.createBucketHandlerPolicy(w, r)
}
func (h *handler) parseCommonCreateBucketParams(reqInfo *middleware.ReqInfo, boxData *accessbox.Box, r *http.Request) (*keys.PublicKey, *layer.CreateBucketParams, error) {
p := &layer.CreateBucketParams{
Name: reqInfo.BucketName,
Namespace: reqInfo.Namespace,
SessionContainerCreation: boxData.Gate.SessionTokenForPut(),
}
if p.SessionContainerCreation == nil {
return nil, nil, fmt.Errorf("%w: couldn't find session token for put", errors.GetAPIError(errors.ErrAccessDenied))
}
if err := checkBucketName(reqInfo.BucketName); err != nil {
return nil, nil, fmt.Errorf("invalid bucket name: %w", err)
}
key, err := getTokenIssuerKey(boxData)
if err != nil {
h.logAndSendError(w, "couldn't get bearer token signature key", reqInfo, err)
return nil, nil, fmt.Errorf("couldn't get bearer token signature key: %w", err)
}
createParams, err := h.parseLocationConstraint(r)
if err != nil {
return nil, nil, fmt.Errorf("could not parse location contraint: %w", err)
}
if err = h.setPlacementPolicy(p, reqInfo.Namespace, createParams.LocationConstraint, boxData.Policies); err != nil {
return nil, nil, fmt.Errorf("couldn't set placement policy: %w", err)
}
p.ObjectLockEnabled = isLockEnabled(r.Header)
return key, p, nil
}
func (h *handler) createBucketHandlerPolicy(w http.ResponseWriter, r *http.Request) {
ctx := r.Context()
reqInfo := middleware.GetReqInfo(ctx)
boxData, err := middleware.GetBoxData(ctx)
if err != nil {
h.logAndSendError(w, "get access box from request", reqInfo, err)
return
}
key, p, err := h.parseCommonCreateBucketParams(reqInfo, boxData, r)
if err != nil {
h.logAndSendError(w, "parse create bucket params", reqInfo, err)
return
}
cannedACL, err := parseCannedACL(r.Header)
if err != nil {
h.logAndSendError(w, "could not parse canned ACL", reqInfo, err)
return
}
p.APEEnabled = true
bktInfo, err := h.obj.CreateBucket(ctx, p)
if err != nil {
h.logAndSendError(w, "could not create bucket", reqInfo, err)
return
}
h.reqLogger(ctx).Info(logs.BucketIsCreated, zap.Stringer("container_id", bktInfo.CID))
chains := bucketCannedACLToAPERules(cannedACL, reqInfo, key, bktInfo.CID)
if err = h.ape.SaveACLChains(reqInfo.Namespace, chains); err != nil {
h.logAndSendError(w, "failed to add morph rule chain", reqInfo, err)
return
}
sp := &layer.PutSettingsParams{
BktInfo: bktInfo,
Settings: &data.BucketSettings{
CannedACL: cannedACL,
OwnerKey: key,
Versioning: data.VersioningUnversioned,
},
}
if p.ObjectLockEnabled {
sp.Settings.Versioning = data.VersioningEnabled
}
if err = h.obj.PutBucketSettings(ctx, sp); err != nil {
h.logAndSendError(w, "couldn't save bucket settings", reqInfo, err,
zap.String("container_id", bktInfo.CID.EncodeToString()))
return
}
middleware.WriteSuccessResponseHeadersOnly(w)
}
func (h *handler) createBucketHandlerACL(w http.ResponseWriter, r *http.Request) {
ctx := r.Context()
reqInfo := middleware.GetReqInfo(ctx)
boxData, err := middleware.GetBoxData(ctx)
if err != nil {
h.logAndSendError(w, "get access box from request", reqInfo, err)
return
}
key, p, err := h.parseCommonCreateBucketParams(reqInfo, boxData, r)
if err != nil {
h.logAndSendError(w, "parse create bucket params", reqInfo, err)
return
}
aclPrm := &layer.PutBucketACLParams{SessionToken: boxData.Gate.SessionTokenForSetEACL()}
if aclPrm.SessionToken == nil {
h.logAndSendError(w, "couldn't find session token for setEACL", reqInfo, errors.GetAPIError(errors.ErrAccessDenied))
return
}
@ -770,67 +894,175 @@ func (h *handler) CreateBucketHandler(w http.ResponseWriter, r *http.Request) {
}
resInfo := &resourceInfo{Bucket: reqInfo.BucketName}
p.EACL, err = bucketACLToTable(bktACL, resInfo)
aclPrm.EACL, err = bucketACLToTable(bktACL, resInfo)
if err != nil {
h.logAndSendError(w, "could translate bucket acl to eacl", reqInfo, err)
return
}
createParams, err := h.parseLocationConstraint(r)
if err != nil {
h.logAndSendError(w, "could not parse body", reqInfo, err)
return
}
var policies []*accessbox.ContainerPolicy
boxData, err := middleware.GetBoxData(ctx)
if err == nil {
policies = boxData.Policies
p.SessionContainerCreation = boxData.Gate.SessionTokenForPut()
p.SessionEACL = boxData.Gate.SessionTokenForSetEACL()
}
if p.SessionContainerCreation == nil {
h.logAndSendError(w, "couldn't find session token for put", reqInfo, errors.GetAPIError(errors.ErrAccessDenied))
return
}
if p.SessionEACL == nil {
h.logAndSendError(w, "couldn't find session token for setEACL", reqInfo, errors.GetAPIError(errors.ErrAccessDenied))
return
}
if err = h.setPolicy(p, reqInfo.Namespace, createParams.LocationConstraint, policies); err != nil {
h.logAndSendError(w, "couldn't set placement policy", reqInfo, err)
return
}
p.ObjectLockEnabled = isLockEnabled(r.Header)
bktInfo, err := h.obj.CreateBucket(ctx, p)
if err != nil {
h.logAndSendError(w, "could not create bucket", reqInfo, err)
return
}
h.reqLogger(ctx).Info(logs.BucketIsCreated, zap.Stringer("container_id", bktInfo.CID))
aclPrm.BktInfo = bktInfo
if err = h.obj.PutBucketACL(r.Context(), aclPrm); err != nil {
h.logAndSendError(w, "could not put bucket e/ACL", reqInfo, err)
return
}
sp := &layer.PutSettingsParams{
BktInfo: bktInfo,
Settings: &data.BucketSettings{
OwnerKey: key,
Versioning: data.VersioningUnversioned,
},
}
if p.ObjectLockEnabled {
sp := &layer.PutSettingsParams{
BktInfo: bktInfo,
Settings: &data.BucketSettings{Versioning: data.VersioningEnabled},
}
if err = h.obj.PutBucketSettings(ctx, sp); err != nil {
h.logAndSendError(w, "couldn't enable bucket versioning", reqInfo, err,
zap.String("container_id", bktInfo.CID.EncodeToString()))
return
}
sp.Settings.Versioning = data.VersioningEnabled
}
if err = h.obj.PutBucketSettings(ctx, sp); err != nil {
h.logAndSendError(w, "couldn't save bucket settings", reqInfo, err,
zap.String("container_id", bktInfo.CID.EncodeToString()))
return
}
middleware.WriteSuccessResponseHeadersOnly(w)
}
func (h handler) setPolicy(prm *layer.CreateBucketParams, namespace, locationConstraint string, userPolicies []*accessbox.ContainerPolicy) error {
const s3ActionPrefix = "s3:"
var (
// https://docs.aws.amazon.com/AmazonS3/latest/userguide/acl-overview.html
writeACLBucketS3Actions = []string{
s3ActionPrefix + middleware.PutObjectOperation,
s3ActionPrefix + middleware.PostObjectOperation,
s3ActionPrefix + middleware.CopyObjectOperation,
s3ActionPrefix + middleware.UploadPartOperation,
s3ActionPrefix + middleware.UploadPartCopyOperation,
s3ActionPrefix + middleware.CreateMultipartUploadOperation,
s3ActionPrefix + middleware.CompleteMultipartUploadOperation,
}
readACLBucketS3Actions = []string{
s3ActionPrefix + middleware.HeadBucketOperation,
s3ActionPrefix + middleware.GetBucketLocationOperation,
s3ActionPrefix + middleware.ListObjectsV1Operation,
s3ActionPrefix + middleware.ListObjectsV2Operation,
s3ActionPrefix + middleware.ListBucketObjectVersionsOperation,
s3ActionPrefix + middleware.ListMultipartUploadsOperation,
}
writeACLBucketNativeActions = []string{
native.MethodPutObject,
}
readACLBucketNativeActions = []string{
native.MethodGetContainer,
native.MethodGetObject,
native.MethodHeadObject,
native.MethodSearchObject,
native.MethodRangeObject,
native.MethodHashObject,
}
)
func bucketCannedACLToAPERules(cannedACL string, reqInfo *middleware.ReqInfo, key *keys.PublicKey, cnrID cid.ID) []*chain.Chain {
cnrIDStr := cnrID.EncodeToString()
chains := []*chain.Chain{
{
ID: getBucketCannedChainID(chain.S3, cnrID),
Rules: []chain.Rule{{
Status: chain.Allow,
Actions: chain.Actions{Names: []string{"s3:*"}},
Resources: chain.Resources{Names: []string{
fmt.Sprintf(s3.ResourceFormatS3Bucket, reqInfo.BucketName),
fmt.Sprintf(s3.ResourceFormatS3BucketObjects, reqInfo.BucketName),
}},
Condition: []chain.Condition{{
Op: chain.CondStringEquals,
Object: chain.ObjectRequest,
Key: s3.PropertyKeyOwner,
Value: key.Address(),
}},
}}},
{
ID: getBucketCannedChainID(chain.Ingress, cnrID),
Rules: []chain.Rule{{
Status: chain.Allow,
Actions: chain.Actions{Names: []string{"*"}},
Resources: chain.Resources{Names: []string{
fmt.Sprintf(native.ResourceFormatNamespaceContainer, reqInfo.Namespace, cnrIDStr),
fmt.Sprintf(native.ResourceFormatNamespaceContainerObjects, reqInfo.Namespace, cnrIDStr),
}},
Condition: []chain.Condition{{
Op: chain.CondStringEquals,
Object: chain.ObjectRequest,
Key: native.PropertyKeyActorPublicKey,
Value: hex.EncodeToString(key.Bytes()),
}},
}},
},
}
switch cannedACL {
case basicACLPrivate:
case cannedACLAuthRead:
fallthrough
case basicACLReadOnly:
chains[0].Rules = append(chains[0].Rules, chain.Rule{
Status: chain.Allow,
Actions: chain.Actions{Names: readACLBucketS3Actions},
Resources: chain.Resources{Names: []string{
fmt.Sprintf(s3.ResourceFormatS3Bucket, reqInfo.BucketName),
fmt.Sprintf(s3.ResourceFormatS3BucketObjects, reqInfo.BucketName),
}},
})
chains[1].Rules = append(chains[1].Rules, chain.Rule{
Status: chain.Allow,
Actions: chain.Actions{Names: readACLBucketNativeActions},
Resources: chain.Resources{Names: []string{
fmt.Sprintf(native.ResourceFormatNamespaceContainer, reqInfo.Namespace, cnrIDStr),
fmt.Sprintf(native.ResourceFormatNamespaceContainerObjects, reqInfo.Namespace, cnrIDStr),
}},
})
case basicACLPublic:
chains[0].Rules = append(chains[0].Rules, chain.Rule{
Status: chain.Allow,
Actions: chain.Actions{Names: append(readACLBucketS3Actions, writeACLBucketS3Actions...)},
Resources: chain.Resources{Names: []string{
fmt.Sprintf(s3.ResourceFormatS3Bucket, reqInfo.BucketName),
fmt.Sprintf(s3.ResourceFormatS3BucketObjects, reqInfo.BucketName),
}},
})
chains[1].Rules = append(chains[1].Rules, chain.Rule{
Status: chain.Allow,
Actions: chain.Actions{Names: append(readACLBucketNativeActions, writeACLBucketNativeActions...)},
Resources: chain.Resources{Names: []string{
fmt.Sprintf(native.ResourceFormatNamespaceContainer, reqInfo.Namespace, cnrIDStr),
fmt.Sprintf(native.ResourceFormatNamespaceContainerObjects, reqInfo.Namespace, cnrIDStr),
}},
})
default:
panic("unknown canned acl") // this should never happen
}
return chains
}
func getBucketCannedChainID(prefix chain.Name, cnrID cid.ID) chain.ID {
return chain.ID(string(prefix) + ":bktCanned" + string(cnrID[:]))
}
func (h handler) setPlacementPolicy(prm *layer.CreateBucketParams, namespace, locationConstraint string, userPolicies []*accessbox.ContainerPolicy) error {
prm.Policy = h.cfg.DefaultPlacementPolicy(namespace)
prm.LocationConstraint = locationConstraint

View file

@ -372,14 +372,28 @@ func TestCreateBucket(t *testing.T) {
hc := prepareHandlerContext(t)
bktName := "bkt-name"
box, _ := createAccessBox(t)
createBucket(t, hc, bktName, box)
createBucketAssertS3Error(hc, bktName, box, s3errors.ErrBucketAlreadyOwnedByYou)
info := createBucket(hc, bktName)
createBucketAssertS3Error(hc, bktName, info.Box, s3errors.ErrBucketAlreadyOwnedByYou)
box2, _ := createAccessBox(t)
createBucketAssertS3Error(hc, bktName, box2, s3errors.ErrBucketAlreadyExists)
}
func TestCreateOldBucketPutVersioning(t *testing.T) {
hc := prepareHandlerContext(t)
hc.config.aclEnabled = true
bktName := "bkt-name"
info := createBucket(hc, bktName)
settings, err := hc.tree.GetSettingsNode(hc.Context(), info.BktInfo)
require.NoError(t, err)
settings.OwnerKey = nil
err = hc.tree.PutSettingsNode(hc.Context(), info.BktInfo, settings)
require.NoError(t, err)
putBucketVersioning(t, hc, bktName, true)
}
func TestCreateNamespacedBucket(t *testing.T) {
hc := prepareHandlerContext(t)
bktName := "bkt-name"

View file

@ -12,6 +12,7 @@ import (
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/acl"
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/eacl"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/session"
@ -62,6 +63,7 @@ func (n *layer) containerInfo(ctx context.Context, idCnr cid.ID) (*data.BucketIn
info.Created = container.CreatedAt(cnr)
info.LocationConstraint = cnr.Attribute(attributeLocationConstraint)
info.HomomorphicHashDisabled = container.IsHomomorphicHashingDisabled(cnr)
info.APEEnabled = cnr.BasicACL().Bits() == 0
attrLockEnabled := cnr.Attribute(AttributeLockEnabled)
if len(attrLockEnabled) > 0 {
@ -119,13 +121,12 @@ func (n *layer) createContainer(ctx context.Context, p *CreateBucketParams) (*da
Created: TimeNow(ctx),
LocationConstraint: p.LocationConstraint,
ObjectLockEnabled: p.ObjectLockEnabled,
APEEnabled: p.APEEnabled,
}
var attributes [][2]string
attributes = append(attributes, [2]string{
attributeLocationConstraint, p.LocationConstraint,
})
attributes := [][2]string{
{attributeLocationConstraint, p.LocationConstraint},
}
if p.ObjectLockEnabled {
attributes = append(attributes, [2]string{
@ -133,6 +134,11 @@ func (n *layer) createContainer(ctx context.Context, p *CreateBucketParams) (*da
})
}
basicACL := acl.PublicRWExtended
if p.APEEnabled {
basicACL = 0
}
res, err := n.frostFS.CreateContainer(ctx, PrmContainerCreate{
Creator: bktInfo.Owner,
Policy: p.Policy,
@ -141,6 +147,7 @@ func (n *layer) createContainer(ctx context.Context, p *CreateBucketParams) (*da
SessionToken: p.SessionContainerCreation,
CreationTime: bktInfo.Created,
AdditionalAttributes: attributes,
BasicACL: basicACL,
})
if err != nil {
return nil, fmt.Errorf("create container: %w", err)
@ -149,10 +156,6 @@ func (n *layer) createContainer(ctx context.Context, p *CreateBucketParams) (*da
bktInfo.CID = res.ContainerID
bktInfo.HomomorphicHashDisabled = res.HomomorphicHashDisabled
if err = n.setContainerEACLTable(ctx, bktInfo.CID, p.EACL, p.SessionEACL); err != nil {
return nil, fmt.Errorf("set container eacl: %w", err)
}
n.cache.PutBucket(bktInfo)
return bktInfo, nil

View file

@ -173,8 +173,6 @@ type FrostFS interface {
// It sets 'Timestamp' attribute to the current time.
// It returns the ID of the saved container.
//
// Created container is public with enabled ACL extension.
//
// It returns exactly one non-zero value. It returns any error encountered which
// prevented the container from being created.
CreateContainer(context.Context, PrmContainerCreate) (*ContainerCreateResult, error)

View file

@ -136,6 +136,10 @@ func (t *TestFrostFS) ContainerID(name string) (cid.ID, error) {
return cid.ID{}, fmt.Errorf("not found")
}
func (t *TestFrostFS) SetContainer(cnrID cid.ID, cnr *container.Container) {
t.containers[cnrID.EncodeToString()] = cnr
}
func (t *TestFrostFS) CreateContainer(_ context.Context, prm PrmContainerCreate) (*ContainerCreateResult, error) {
var cnr container.Container
cnr.Init()

View file

@ -175,11 +175,10 @@ type (
Name string
Namespace string
Policy netmap.PlacementPolicy
EACL *eacl.Table
SessionContainerCreation *session.Container
SessionEACL *session.Container
LocationConstraint string
ObjectLockEnabled bool
APEEnabled bool
}
// PutBucketACLParams stores put bucket acl request parameters.
PutBucketACLParams struct {

View file

@ -1,6 +1,7 @@
package middleware
import (
"context"
"crypto/elliptic"
"fmt"
"net/http"
@ -19,27 +20,29 @@ import (
)
type PolicySettings interface {
ResolveNamespaceAlias(ns string) string
PolicyDenyByDefault() bool
ACLEnabled() bool
}
type FrostFSIDInformer interface {
GetUserGroupIDs(userHash util.Uint160) ([]string, error)
}
func PolicyCheck(storage engine.ChainRouter, frostfsid FrostFSIDInformer, settings PolicySettings, domains []string, log *zap.Logger) Func {
type PolicyConfig struct {
Storage engine.ChainRouter
FrostfsID FrostFSIDInformer
Settings PolicySettings
Domains []string
Log *zap.Logger
BucketResolver BucketResolveFunc
}
func PolicyCheck(cfg PolicyConfig) Func {
return func(h http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
ctx := r.Context()
st, err := policyCheck(storage, frostfsid, settings, domains, r)
if err == nil {
if st != chain.Allow && (st != chain.NoRuleFound || settings.PolicyDenyByDefault()) {
err = apiErr.GetAPIErrorWithError(apiErr.ErrAccessDenied, fmt.Errorf("policy check: %s", st.String()))
}
}
if err != nil {
reqLogOrDefault(ctx, log).Error(logs.PolicyValidationFailed, zap.Error(err))
if err := policyCheck(r, cfg); err != nil {
reqLogOrDefault(ctx, cfg.Log).Error(logs.PolicyValidationFailed, zap.Error(err))
WriteErrorResponse(w, GetReqInfo(ctx), err)
return
}
@ -49,27 +52,58 @@ func PolicyCheck(storage engine.ChainRouter, frostfsid FrostFSIDInformer, settin
}
}
func policyCheck(storage engine.ChainRouter, frostfsid FrostFSIDInformer, settings PolicySettings, domains []string, r *http.Request) (chain.Status, error) {
req, err := getPolicyRequest(r, frostfsid, domains)
func policyCheck(r *http.Request, cfg PolicyConfig) error {
reqType, bktName, objName := getBucketObject(r, cfg.Domains)
req, err := getPolicyRequest(r, cfg.FrostfsID, reqType, bktName, objName)
if err != nil {
return 0, err
return err
}
reqInfo := GetReqInfo(r.Context())
target := engine.NewRequestTargetWithNamespace(settings.ResolveNamespaceAlias(reqInfo.Namespace))
st, found, err := storage.IsAllowed(chain.S3, target, req)
target := engine.NewRequestTargetWithNamespace(reqInfo.Namespace)
st, found, err := cfg.Storage.IsAllowed(chain.S3, target, req)
if err != nil {
return 0, err
return err
}
if !found {
st = chain.NoRuleFound
}
return st, nil
switch {
case st == chain.Allow:
return nil
case st != chain.NoRuleFound:
return apiErr.GetAPIErrorWithError(apiErr.ErrAccessDenied, fmt.Errorf("policy check: %s", st.String()))
}
isAPE, err := isAPEBehavior(r.Context(), req, cfg, reqType, bktName)
if err != nil {
return err
}
if isAPE && cfg.Settings.PolicyDenyByDefault() {
return apiErr.GetAPIErrorWithError(apiErr.ErrAccessDenied, fmt.Errorf("policy check: %s", st.String()))
}
return nil
}
func getPolicyRequest(r *http.Request, frostfsid FrostFSIDInformer, domains []string) (*testutil.Request, error) {
func isAPEBehavior(ctx context.Context, req *testutil.Request, cfg PolicyConfig, reqType ReqType, bktName string) (bool, error) {
if reqType == noneType ||
strings.HasSuffix(req.Operation(), CreateBucketOperation) {
return !cfg.Settings.ACLEnabled(), nil
}
bktInfo, err := cfg.BucketResolver(ctx, bktName) // we cannot use reqInfo.BucketName because it hasn't been set yet
if err != nil {
return false, err
}
return bktInfo.APEEnabled, nil
}
func getPolicyRequest(r *http.Request, frostfsid FrostFSIDInformer, reqType ReqType, bktName string, objName string) (*testutil.Request, error) {
var (
owner string
groups []string
@ -90,7 +124,14 @@ func getPolicyRequest(r *http.Request, frostfsid FrostFSIDInformer, domains []st
}
}
op, res := determineOperationAndResource(r, domains)
op := determineOperation(r, reqType)
var res string
switch reqType {
case objectType:
res = fmt.Sprintf(s3.ResourceFormatS3BucketObject, bktName, objName)
default:
res = fmt.Sprintf(s3.ResourceFormatS3Bucket, bktName)
}
return testutil.NewRequest(op, testutil.NewResource(res, nil),
map[string]string{
@ -108,45 +149,34 @@ const (
objectType
)
func determineOperationAndResource(r *http.Request, domains []string) (operation string, resource string) {
var (
reqType ReqType
matchDomain bool
)
func getBucketObject(r *http.Request, domains []string) (reqType ReqType, bktName string, objName string) {
for _, domain := range domains {
ind := strings.Index(r.Host, "."+domain)
if ind == -1 {
continue
}
matchDomain = true
reqType = bucketType
bkt := r.Host[:ind]
if obj := strings.TrimPrefix(r.URL.Path, "/"); obj != "" {
reqType = objectType
resource = fmt.Sprintf(s3.ResourceFormatS3BucketObject, bkt, obj)
} else {
resource = fmt.Sprintf(s3.ResourceFormatS3Bucket, bkt)
return objectType, bkt, obj
}
break
return bucketType, bkt, ""
}
if !matchDomain {
bktObj := strings.TrimPrefix(r.URL.Path, "/")
if ind := strings.IndexByte(bktObj, '/'); ind == -1 {
reqType = bucketType
resource = fmt.Sprintf(s3.ResourceFormatS3Bucket, bktObj)
if bktObj == "" {
reqType = noneType
}
} else {
reqType = objectType
resource = fmt.Sprintf(s3.ResourceFormatS3BucketObject, bktObj[:ind], bktObj[ind+1:])
}
bktObj := strings.TrimPrefix(r.URL.Path, "/")
if bktObj == "" {
return noneType, "", ""
}
if ind := strings.IndexByte(bktObj, '/'); ind != -1 {
return objectType, bktObj[:ind], bktObj[ind+1:]
}
return bucketType, bktObj, ""
}
func determineOperation(r *http.Request, reqType ReqType) (operation string) {
switch reqType {
case objectType:
operation = determineObjectOperation(r)
@ -156,7 +186,7 @@ func determineOperationAndResource(r *http.Request, domains []string) (operation
operation = determineGeneralOperation(r)
}
return "s3:" + operation, resource
return "s3:" + operation
}
func determineBucketOperation(r *http.Request) string {

View file

@ -191,6 +191,7 @@ func GetReqLog(ctx context.Context) *zap.Logger {
type RequestSettings interface {
NamespaceHeader() string
ResolveNamespaceAlias(string) string
}
func Request(log *zap.Logger, settings RequestSettings) Func {
@ -207,7 +208,7 @@ func Request(log *zap.Logger, settings RequestSettings) Func {
// set request info into context
// bucket name and object will be set in reqInfo later (limitation of go-chi)
reqInfo := NewReqInfo(w, r, ObjectRequest{})
reqInfo.Namespace = r.Header.Get(settings.NamespaceHeader())
reqInfo.Namespace = settings.ResolveNamespaceAlias(r.Header.Get(settings.NamespaceHeader()))
r = r.WithContext(SetReqInfo(r.Context(), reqInfo))
// set request id into gRPC meta header

View file

@ -136,7 +136,14 @@ func NewRouter(cfg Config) *chi.Mux {
api.Use(s3middleware.FrostfsIDValidation(cfg.FrostfsID, cfg.Log))
}
api.Use(s3middleware.PolicyCheck(cfg.PolicyChecker, cfg.FrostfsID, cfg.MiddlewareSettings, cfg.Domains, cfg.Log))
api.Use(s3middleware.PolicyCheck(s3middleware.PolicyConfig{
Storage: cfg.PolicyChecker,
FrostfsID: cfg.FrostfsID,
Settings: cfg.MiddlewareSettings,
Domains: cfg.Domains,
Log: cfg.Log,
BucketResolver: cfg.Handler.ResolveBucket,
}))
defaultRouter := chi.NewRouter()
defaultRouter.Mount(fmt.Sprintf("/{%s}", s3middleware.BucketURLPrm), bucketRouter(cfg.Handler, cfg.Log))

View file

@ -3,6 +3,7 @@ package api
import (
"context"
"encoding/json"
"errors"
"net/http"
"testing"
@ -53,6 +54,7 @@ func (c *centerMock) Authenticate(*http.Request) (*middleware.Box, error) {
type middlewareSettingsMock struct {
denyByDefault bool
aclEnabled bool
}
func (r *middlewareSettingsMock) NamespaceHeader() string {
@ -67,6 +69,10 @@ func (r *middlewareSettingsMock) PolicyDenyByDefault() bool {
return r.denyByDefault
}
func (r *middlewareSettingsMock) ACLEnabled() bool {
return r.aclEnabled
}
type frostFSIDMock struct {
}
@ -79,7 +85,9 @@ func (f *frostFSIDMock) GetUserGroupIDs(util.Uint160) ([]string, error) {
}
type handlerMock struct {
t *testing.T
t *testing.T
cfg *middlewareSettingsMock
buckets map[string]*data.BucketInfo
}
type handlerResult struct {
@ -339,9 +347,20 @@ func (h *handlerMock) PutBucketNotificationHandler(http.ResponseWriter, *http.Re
panic("implement me")
}
func (h *handlerMock) CreateBucketHandler(http.ResponseWriter, *http.Request) {
//TODO implement me
panic("implement me")
func (h *handlerMock) CreateBucketHandler(w http.ResponseWriter, r *http.Request) {
reqInfo := middleware.GetReqInfo(r.Context())
h.buckets[reqInfo.Namespace+reqInfo.BucketName] = &data.BucketInfo{
Name: reqInfo.BucketName,
APEEnabled: !h.cfg.ACLEnabled(),
}
res := &handlerResult{
Method: middleware.CreateBucketOperation,
ReqInfo: middleware.GetReqInfo(r.Context()),
}
h.writeResponse(w, res)
}
func (h *handlerMock) HeadBucketHandler(w http.ResponseWriter, r *http.Request) {
@ -443,8 +462,13 @@ func (h *handlerMock) ListMultipartUploadsHandler(w http.ResponseWriter, r *http
h.writeResponse(w, res)
}
func (h *handlerMock) ResolveBucket(context.Context, string) (*data.BucketInfo, error) {
return &data.BucketInfo{}, nil
func (h *handlerMock) ResolveBucket(ctx context.Context, name string) (*data.BucketInfo, error) {
reqInfo := middleware.GetReqInfo(ctx)
bktInfo, ok := h.buckets[reqInfo.Namespace+name]
if !ok {
return nil, errors.New("not found")
}
return bktInfo, nil
}
func (h *handlerMock) writeResponse(w http.ResponseWriter, resp *handlerResult) {

View file

@ -11,6 +11,7 @@ import (
"testing"
"time"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
apiErrors "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
s3middleware "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/middleware"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/metrics"
@ -27,6 +28,7 @@ import (
)
type routerMock struct {
t *testing.T
router *chi.Mux
cfg Config
middlewareSettings *middlewareSettingsMock
@ -55,7 +57,7 @@ func prepareRouter(t *testing.T) *routerMock {
Limit: 10,
BacklogTimeout: 30 * time.Second,
},
Handler: &handlerMock{t: t},
Handler: &handlerMock{t: t, cfg: middlewareSettings, buckets: map[string]*data.BucketInfo{}},
Center: &centerMock{t: t},
Log: logger,
Metrics: metrics.NewAppMetrics(metricsConfig),
@ -65,6 +67,7 @@ func prepareRouter(t *testing.T) *routerMock {
FrostfsID: &frostFSIDMock{},
}
return &routerMock{
t: t,
router: NewRouter(cfg),
cfg: cfg,
middlewareSettings: middlewareSettings,
@ -75,6 +78,8 @@ func prepareRouter(t *testing.T) *routerMock {
func TestRouterUploadPart(t *testing.T) {
chiRouter := prepareRouter(t)
createBucket(chiRouter, "", "dkirillov")
w := httptest.NewRecorder()
r := httptest.NewRequest(http.MethodPut, "/dkirillov/fix-object", nil)
query := make(url.Values)
@ -90,6 +95,8 @@ func TestRouterUploadPart(t *testing.T) {
func TestRouterListMultipartUploads(t *testing.T) {
chiRouter := prepareRouter(t)
createBucket(chiRouter, "", "test-bucket")
w := httptest.NewRecorder()
r := httptest.NewRequest(http.MethodGet, "/test-bucket", nil)
query := make(url.Values)
@ -104,22 +111,18 @@ func TestRouterListMultipartUploads(t *testing.T) {
func TestRouterObjectWithSlashes(t *testing.T) {
chiRouter := prepareRouter(t)
bktName, objName := "dkirillov", "/fix/object"
target := fmt.Sprintf("/%s/%s", bktName, objName)
ns, bktName, objName := "", "dkirillov", "/fix/object"
w := httptest.NewRecorder()
r := httptest.NewRequest(http.MethodPut, target, nil)
chiRouter.ServeHTTP(w, r)
resp := readResponse(t, w)
require.Equal(t, "PutObject", resp.Method)
createBucket(chiRouter, ns, bktName)
resp := putObject(chiRouter, ns, bktName, objName)
require.Equal(t, objName, resp.ReqInfo.ObjectName)
}
func TestRouterObjectEscaping(t *testing.T) {
chiRouter := prepareRouter(t)
bktName := "dkirillov"
ns, bktName := "", "dkirillov"
createBucket(chiRouter, ns, bktName)
for _, tc := range []struct {
name string
@ -153,14 +156,7 @@ func TestRouterObjectEscaping(t *testing.T) {
},
} {
t.Run(tc.name, func(t *testing.T) {
target := fmt.Sprintf("/%s/%s", bktName, tc.objName)
w := httptest.NewRecorder()
r := httptest.NewRequest(http.MethodPut, target, nil)
chiRouter.ServeHTTP(w, r)
resp := readResponse(t, w)
require.Equal(t, "PutObject", resp.Method)
resp := putObject(chiRouter, ns, bktName, tc.objName)
require.Equal(t, tc.expectedObjName, resp.ReqInfo.ObjectName)
})
}
@ -168,40 +164,33 @@ func TestRouterObjectEscaping(t *testing.T) {
func TestPolicyChecker(t *testing.T) {
chiRouter := prepareRouter(t)
namespace := "custom-ns"
bktName, objName := "bucket", "object"
target := fmt.Sprintf("/%s/%s", bktName, objName)
ns1, bktName1, objName1 := "", "bucket", "object"
ns2, bktName2, objName2 := "custom-ns", "other-bucket", "object"
createBucket(chiRouter, ns1, bktName1)
createBucket(chiRouter, ns2, bktName1)
createBucket(chiRouter, ns2, bktName2)
ruleChain := &chain.Chain{
ID: chain.ID("id"),
Rules: []chain.Rule{{
Status: chain.AccessDenied,
Actions: chain.Actions{Names: []string{"*"}},
Resources: chain.Resources{Names: []string{fmt.Sprintf(s3.ResourceFormatS3BucketObjects, bktName)}},
Resources: chain.Resources{Names: []string{fmt.Sprintf(s3.ResourceFormatS3BucketObjects, bktName1)}},
}},
}
_, _, err := chiRouter.policyChecker.MorphRuleChainStorage().AddMorphRuleChain(chain.S3, engine.NamespaceTarget(namespace), ruleChain)
_, _, err := chiRouter.policyChecker.MorphRuleChainStorage().AddMorphRuleChain(chain.S3, engine.NamespaceTarget(ns2), ruleChain)
require.NoError(t, err)
// check we can access 'bucket' in default namespace
w, r := httptest.NewRecorder(), httptest.NewRequest(http.MethodPut, target, nil)
chiRouter.ServeHTTP(w, r)
resp := readResponse(t, w)
require.Equal(t, s3middleware.PutObjectOperation, resp.Method)
putObject(chiRouter, ns1, bktName1, objName1)
// check we can access 'other-bucket' in custom namespace
w, r = httptest.NewRecorder(), httptest.NewRequest(http.MethodPut, "/other-bucket/object", nil)
r.Header.Set(FrostfsNamespaceHeader, namespace)
chiRouter.ServeHTTP(w, r)
resp = readResponse(t, w)
require.Equal(t, s3middleware.PutObjectOperation, resp.Method)
putObject(chiRouter, ns2, bktName2, objName2)
// check we cannot access 'bucket' in custom namespace
w, r = httptest.NewRecorder(), httptest.NewRequest(http.MethodPut, target, nil)
r.Header.Set(FrostfsNamespaceHeader, namespace)
chiRouter.ServeHTTP(w, r)
assertAPIError(t, w, apiErrors.ErrAccessDenied)
putObjectErr(chiRouter, ns2, bktName1, objName2, apiErrors.ErrAccessDenied)
}
func TestPolicyCheckerReqTypeDetermination(t *testing.T) {
@ -248,52 +237,229 @@ func TestPolicyCheckerReqTypeDetermination(t *testing.T) {
func TestDefaultBehaviorPolicyChecker(t *testing.T) {
chiRouter := prepareRouter(t)
bktName, objName := "bucket", "object"
target := fmt.Sprintf("/%s/%s", bktName, objName)
ns, bktName := "", "bucket"
// check we can access bucket if rules not found
w, r := httptest.NewRecorder(), httptest.NewRequest(http.MethodPut, target, nil)
chiRouter.ServeHTTP(w, r)
resp := readResponse(t, w)
require.Equal(t, s3middleware.PutObjectOperation, resp.Method)
createBucket(chiRouter, ns, bktName)
// check we cannot access if rules not found when settings is enabled
chiRouter.middlewareSettings.denyByDefault = true
w, r = httptest.NewRecorder(), httptest.NewRequest(http.MethodPut, target, nil)
chiRouter.ServeHTTP(w, r)
assertAPIError(t, w, apiErrors.ErrAccessDenied)
createBucketErr(chiRouter, ns, bktName, apiErrors.ErrAccessDenied)
}
func TestACLAPE(t *testing.T) {
t.Run("acl disabled, ape deny by default", func(t *testing.T) {
router := prepareRouter(t)
ns, bktName, objName := "", "bucket", "object"
bktNameOld, bktNameNew := "old-bucket", "new-bucket"
createOldBucket(router, bktNameOld)
createNewBucket(router, bktNameNew)
router.middlewareSettings.aclEnabled = false
router.middlewareSettings.denyByDefault = true
// Allow because of using old bucket
putObject(router, ns, bktNameOld, objName)
// Deny because of deny by default
putObjectErr(router, ns, bktNameNew, objName, apiErrors.ErrAccessDenied)
// Deny because of deny by default
createBucketErr(router, ns, bktName, apiErrors.ErrAccessDenied)
listBucketsErr(router, ns, apiErrors.ErrAccessDenied)
// Allow operations and check
allowOperations(router, ns, []string{"s3:CreateBucket", "s3:ListBuckets"})
createBucket(router, ns, bktName)
listBuckets(router, ns)
})
t.Run("acl disabled, ape allow by default", func(t *testing.T) {
router := prepareRouter(t)
ns, bktName, objName := "", "bucket", "object"
bktNameOld, bktNameNew := "old-bucket", "new-bucket"
createOldBucket(router, bktNameOld)
createNewBucket(router, bktNameNew)
router.middlewareSettings.aclEnabled = false
router.middlewareSettings.denyByDefault = false
// Allow because of using old bucket
putObject(router, ns, bktNameOld, objName)
// Allow because of allow by default
putObject(router, ns, bktNameNew, objName)
// Allow because of deny by default
createBucket(router, ns, bktName)
listBuckets(router, ns)
// Deny operations and check
denyOperations(router, ns, []string{"s3:CreateBucket", "s3:ListBuckets"})
createBucketErr(router, ns, bktName, apiErrors.ErrAccessDenied)
listBucketsErr(router, ns, apiErrors.ErrAccessDenied)
})
t.Run("acl enabled, ape deny by default", func(t *testing.T) {
router := prepareRouter(t)
ns, bktName, objName := "", "bucket", "object"
bktNameOld, bktNameNew := "old-bucket", "new-bucket"
createOldBucket(router, bktNameOld)
createNewBucket(router, bktNameNew)
router.middlewareSettings.aclEnabled = true
router.middlewareSettings.denyByDefault = true
// Allow because of using old bucket
putObject(router, ns, bktNameOld, objName)
// Deny because of deny by default
putObjectErr(router, ns, bktNameNew, objName, apiErrors.ErrAccessDenied)
// Allow because of old behavior
createBucket(router, ns, bktName)
listBuckets(router, ns)
})
t.Run("acl enabled, ape allow by default", func(t *testing.T) {
router := prepareRouter(t)
ns, bktName, objName := "", "bucket", "object"
bktNameOld, bktNameNew := "old-bucket", "new-bucket"
createOldBucket(router, bktNameOld)
createNewBucket(router, bktNameNew)
router.middlewareSettings.aclEnabled = true
router.middlewareSettings.denyByDefault = false
// Allow because of using old bucket
putObject(router, ns, bktNameOld, objName)
// Allow because of allow by default
putObject(router, ns, bktNameNew, objName)
// Allow because of old behavior
createBucket(router, ns, bktName)
listBuckets(router, ns)
})
}
func allowOperations(router *routerMock, ns string, operations []string) {
addPolicy(router, ns, "allow", engineiam.AllowEffect, operations)
}
func denyOperations(router *routerMock, ns string, operations []string) {
addPolicy(router, ns, "deny", engineiam.DenyEffect, operations)
}
func addPolicy(router *routerMock, ns string, id string, effect engineiam.Effect, operations []string) {
policy := engineiam.Policy{
Statement: []engineiam.Statement{{
Principal: map[engineiam.PrincipalType][]string{engineiam.Wildcard: {}},
Effect: effect,
Action: engineiam.Action(operations),
Resource: engineiam.Resource{fmt.Sprintf(s3.ResourceFormatS3All)},
}},
}
ruleChain, err := engineiam.ConvertToS3Chain(policy, nil)
require.NoError(router.t, err)
ruleChain.ID = chain.ID(id)
_, _, err = router.policyChecker.MorphRuleChainStorage().AddMorphRuleChain(chain.S3, engine.NamespaceTarget(ns), ruleChain)
require.NoError(router.t, err)
}
func createOldBucket(router *routerMock, bktName string) {
createSpecificBucket(router, bktName, true)
}
func createNewBucket(router *routerMock, bktName string) {
createSpecificBucket(router, bktName, false)
}
func createSpecificBucket(router *routerMock, bktName string, old bool) {
aclEnabled := router.middlewareSettings.ACLEnabled()
router.middlewareSettings.aclEnabled = old
createBucket(router, "", bktName)
router.middlewareSettings.aclEnabled = aclEnabled
}
func createBucket(router *routerMock, namespace, bktName string) {
w := createBucketBase(router, namespace, bktName)
resp := readResponse(router.t, w)
require.Equal(router.t, s3middleware.CreateBucketOperation, resp.Method)
}
func createBucketErr(router *routerMock, namespace, bktName string, errCode apiErrors.ErrorCode) {
w := createBucketBase(router, namespace, bktName)
assertAPIError(router.t, w, errCode)
}
func createBucketBase(router *routerMock, namespace, bktName string) *httptest.ResponseRecorder {
w, r := httptest.NewRecorder(), httptest.NewRequest(http.MethodPut, "/"+bktName, nil)
r.Header.Set(FrostfsNamespaceHeader, namespace)
router.ServeHTTP(w, r)
return w
}
func listBuckets(router *routerMock, namespace string) {
w := listBucketsBase(router, namespace)
resp := readResponse(router.t, w)
require.Equal(router.t, s3middleware.ListBucketsOperation, resp.Method)
}
func listBucketsErr(router *routerMock, namespace string, errCode apiErrors.ErrorCode) {
w := listBucketsBase(router, namespace)
assertAPIError(router.t, w, errCode)
}
func listBucketsBase(router *routerMock, namespace string) *httptest.ResponseRecorder {
w, r := httptest.NewRecorder(), httptest.NewRequest(http.MethodGet, "/", nil)
r.Header.Set(FrostfsNamespaceHeader, namespace)
router.ServeHTTP(w, r)
return w
}
func putObject(router *routerMock, namespace, bktName, objName string) handlerResult {
w := putObjectBase(router, namespace, bktName, objName)
resp := readResponse(router.t, w)
require.Equal(router.t, s3middleware.PutObjectOperation, resp.Method)
return resp
}
func putObjectErr(router *routerMock, namespace, bktName, objName string, errCode apiErrors.ErrorCode) {
w := putObjectBase(router, namespace, bktName, objName)
assertAPIError(router.t, w, errCode)
}
func putObjectBase(router *routerMock, namespace, bktName, objName string) *httptest.ResponseRecorder {
w, r := httptest.NewRecorder(), httptest.NewRequest(http.MethodPut, "/"+bktName+"/"+objName, nil)
r.Header.Set(FrostfsNamespaceHeader, namespace)
router.ServeHTTP(w, r)
return w
}
func TestOwnerIDRetrieving(t *testing.T) {
chiRouter := prepareRouter(t)
w := httptest.NewRecorder()
r := httptest.NewRequest(http.MethodGet, "/test-bucket", nil)
ns, bktName, objName := "", "test-bucket", "test-object"
chiRouter.ServeHTTP(w, r)
resp := readResponse(t, w)
createBucket(chiRouter, ns, bktName)
resp := putObject(chiRouter, ns, bktName, objName)
require.NotEqual(t, "anon", resp.ReqInfo.User)
w = httptest.NewRecorder()
r = httptest.NewRequest(http.MethodGet, "/test-bucket", nil)
chiRouter.cfg.Center.(*centerMock).anon = true
chiRouter.ServeHTTP(w, r)
resp = readResponse(t, w)
resp = putObject(chiRouter, ns, bktName, objName)
require.Equal(t, "anon", resp.ReqInfo.User)
}
func TestBillingMetrics(t *testing.T) {
chiRouter := prepareRouter(t)
bktName, objName := "test-bucket", "test-object"
target := fmt.Sprintf("/%s/%s", bktName, objName)
ns, bktName, objName := "", "test-bucket", "test-object"
w := httptest.NewRecorder()
r := httptest.NewRequest(http.MethodPut, target, nil)
chiRouter.ServeHTTP(w, r)
createBucket(chiRouter, ns, bktName)
dump := chiRouter.cfg.Metrics.UsersAPIStats().DumpMetrics()
require.Len(t, dump.Requests, 1)
require.NotEqual(t, "anon", dump.Requests[0].User)
@ -302,11 +468,7 @@ func TestBillingMetrics(t *testing.T) {
require.Equal(t, 1, dump.Requests[0].Requests)
chiRouter.cfg.Center.(*centerMock).anon = true
w = httptest.NewRecorder()
r = httptest.NewRequest(http.MethodPut, target, nil)
chiRouter.ServeHTTP(w, r)
putObject(chiRouter, ns, bktName, objName)
dump = chiRouter.cfg.Metrics.UsersAPIStats().DumpMetrics()
require.Len(t, dump.Requests, 1)
require.Equal(t, "anon", dump.Requests[0].User)

View file

@ -100,6 +100,7 @@ type (
clientCut bool
maxBufferSizeForPut uint64
md5Enabled bool
aclEnabled bool
namespaceHeader string
defaultNamespaces []string
authorizedControlAPIKeys [][]byte
@ -220,18 +221,29 @@ func newAppSettings(log *Logger, v *viper.Viper, key *keys.PrivateKey) *appSetti
}
func (s *appSettings) update(v *viper.Viper, log *zap.Logger, key *keys.PrivateKey) {
s.setNamespaceHeader(v.GetString(cfgResolveNamespaceHeader)) // should be updated before placement policies
s.initPlacementPolicy(log, v)
s.updateNamespacesSettings(v, log)
s.useDefaultXMLNamespace(v.GetBool(cfgKludgeUseDefaultXMLNS))
s.setACLEnabled(v.GetBool(cfgKludgeACLEnabled))
s.setBypassContentEncodingInChunks(v.GetBool(cfgKludgeBypassContentEncodingCheckInChunks))
s.setClientCut(v.GetBool(cfgClientCut))
s.setBufferMaxSizeForPut(v.GetUint64(cfgBufferMaxSizeForPut))
s.setMD5Enabled(v.GetBool(cfgMD5Enabled))
s.setDefaultNamespaces(fetchDefaultNamespaces(log, v))
s.setAuthorizedControlAPIKeys(append(fetchAuthorizedKeys(log, v), key.PublicKey()))
s.setPolicyDenyByDefault(v.GetBool(cfgPolicyDenyByDefault))
}
func (s *appSettings) updateNamespacesSettings(v *viper.Viper, log *zap.Logger) {
nsHeader := v.GetString(cfgResolveNamespaceHeader)
nsConfig, defaultNamespaces := fetchNamespacesConfig(log, v)
s.mu.Lock()
defer s.mu.Unlock()
s.namespaceHeader = nsHeader
s.defaultNamespaces = defaultNamespaces
s.namespaces = nsConfig.Namespaces
}
func (s *appSettings) BypassContentEncodingInChunks() bool {
s.mu.RLock()
defer s.mu.RUnlock()
@ -268,15 +280,6 @@ func (s *appSettings) setBufferMaxSizeForPut(size uint64) {
s.mu.Unlock()
}
func (s *appSettings) initPlacementPolicy(l *zap.Logger, v *viper.Viper) {
nsConfig := fetchNamespacesConfig(l, v)
s.mu.Lock()
defer s.mu.Unlock()
s.namespaces = nsConfig.Namespaces
}
func (s *appSettings) DefaultPlacementPolicy(namespace string) netmap.PlacementPolicy {
s.mu.RLock()
defer s.mu.RUnlock()
@ -351,39 +354,39 @@ func (s *appSettings) setMD5Enabled(md5Enabled bool) {
s.mu.Unlock()
}
func (s *appSettings) setACLEnabled(enableACL bool) {
s.mu.Lock()
s.aclEnabled = enableACL
s.mu.Unlock()
}
func (s *appSettings) ACLEnabled() bool {
s.mu.RLock()
defer s.mu.RUnlock()
return s.aclEnabled
}
func (s *appSettings) NamespaceHeader() string {
s.mu.RLock()
defer s.mu.RUnlock()
return s.namespaceHeader
}
func (s *appSettings) setNamespaceHeader(nsHeader string) {
s.mu.Lock()
s.namespaceHeader = nsHeader
s.mu.Unlock()
}
func (s *appSettings) FormContainerZone(ns string) (zone string, isDefault bool) {
if s.IsDefaultNamespace(ns) {
if len(ns) == 0 {
return v2container.SysAttributeZoneDefault, true
}
return ns + ".ns", false
}
func (s *appSettings) IsDefaultNamespace(ns string) bool {
func (s *appSettings) isDefaultNamespace(ns string) bool {
s.mu.RLock()
namespaces := s.defaultNamespaces
s.mu.RUnlock()
return slices.Contains(namespaces, ns)
}
func (s *appSettings) setDefaultNamespaces(namespaces []string) {
s.mu.Lock()
s.defaultNamespaces = namespaces
s.mu.Unlock()
}
func (s *appSettings) FetchRawKeys() [][]byte {
s.mu.RLock()
defer s.mu.RUnlock()
@ -402,7 +405,7 @@ func (s *appSettings) setAuthorizedControlAPIKeys(keys keys.PublicKeys) {
}
func (s *appSettings) ResolveNamespaceAlias(namespace string) string {
if s.IsDefaultNamespace(namespace) {
if s.isDefaultNamespace(namespace) {
return defaultNamespace
}
@ -463,23 +466,14 @@ func (a *App) initFrostfsID(ctx context.Context) {
}
func (a *App) initPolicyStorage(ctx context.Context) {
var (
err error
policyContract policy.Contract
)
if a.cfg.GetBool(cfgPolicyEnabled) {
policyContract, err = contract.New(ctx, contract.Config{
RPCAddress: a.cfg.GetString(cfgRPCEndpoint),
Contract: a.cfg.GetString(cfgPolicyContract),
ProxyContract: a.cfg.GetString(cfgProxyContract),
Key: a.key,
})
if err != nil {
a.log.Fatal(logs.InitPolicyContractFailed, zap.Error(err))
}
} else {
policyContract = contract.NewInMemoryContract()
policyContract, err := contract.New(ctx, contract.Config{
RPCAddress: a.cfg.GetString(cfgRPCEndpoint),
Contract: a.cfg.GetString(cfgPolicyContract),
ProxyContract: a.cfg.GetString(cfgProxyContract),
Key: a.key,
})
if err != nil {
a.log.Fatal(logs.InitPolicyContractFailed, zap.Error(err))
}
a.policyStorage = policy.NewStorage(policy.StorageConfig{
@ -957,16 +951,9 @@ func getMorphPolicyCacheConfig(v *viper.Viper, l *zap.Logger) *cache.Config {
}
func (a *App) initHandler() {
var (
err error
ffsid handler.FrostFSID
)
var err error
if a.frostfsid != nil {
ffsid = a.frostfsid
}
a.api, err = handler.New(a.log, a.obj, a.nc, a.settings, a.policyStorage, ffsid)
a.api, err = handler.New(a.log, a.obj, a.nc, a.settings, a.policyStorage, a.frostfsid)
if err != nil {
a.log.Fatal(logs.CouldNotInitializeAPIHandler, zap.Error(err))
}

View file

@ -166,6 +166,7 @@ const ( // Settings.
cfgKludgeUseDefaultXMLNS = "kludge.use_default_xmlns"
cfgKludgeBypassContentEncodingCheckInChunks = "kludge.bypass_content_encoding_check_in_chunks"
cfgKludgeDefaultNamespaces = "kludge.default_namespaces"
cfgKludgeACLEnabled = "kludge.acl_enabled"
// Web.
cfgWebReadTimeout = "web.read_timeout"
@ -216,7 +217,6 @@ const ( // Settings.
cfgFrostfsIDValidationEnabled = "frostfsid.validation.enabled"
// Policy.
cfgPolicyEnabled = "policy.enabled"
cfgPolicyContract = "policy.contract"
// Proxy.
@ -515,7 +515,7 @@ func fetchDefaultNamespaces(l *zap.Logger, v *viper.Viper) []string {
return defaultNamespaces
}
func fetchNamespacesConfig(l *zap.Logger, v *viper.Viper) NamespacesConfig {
func fetchNamespacesConfig(l *zap.Logger, v *viper.Viper) (NamespacesConfig, []string) {
defaultNSRegionMap := fetchRegionMappingPolicies(l, v)
defaultNSRegionMap[defaultConstraintName] = fetchDefaultPolicy(l, v)
@ -551,15 +551,13 @@ func fetchNamespacesConfig(l *zap.Logger, v *viper.Viper) NamespacesConfig {
}
}
for _, name := range defaultNamespacesNames {
nsConfig.Namespaces[name] = Namespace{
Name: name,
LocationConstraints: defaultNSValue.LocationConstraints,
CopiesNumbers: defaultNSValue.CopiesNumbers,
}
nsConfig.Namespaces[defaultNamespace] = Namespace{
Name: defaultNamespace,
LocationConstraints: defaultNSValue.LocationConstraints,
CopiesNumbers: defaultNSValue.CopiesNumbers,
}
return nsConfig
return nsConfig, defaultNamespacesNames
}
func readNamespacesConfig(filepath string) (NamespacesConfig, error) {
@ -719,6 +717,7 @@ func newSettings() *viper.Viper {
v.SetDefault(cfgKludgeUseDefaultXMLNS, false)
v.SetDefault(cfgKludgeBypassContentEncodingCheckInChunks, false)
v.SetDefault(cfgKludgeDefaultNamespaces, defaultDefaultNamespaces)
v.SetDefault(cfgKludgeACLEnabled, false)
// web
v.SetDefault(cfgWebReadHeaderTimeout, defaultReadHeaderTimeout)
@ -729,7 +728,6 @@ func newSettings() *viper.Viper {
// policy
v.SetDefault(cfgPolicyContract, "policy.frostfs")
v.SetDefault(cfgPolicyEnabled, true)
// proxy
v.SetDefault(cfgProxyContract, "proxy.frostfs")

View file

@ -162,6 +162,8 @@ S3_GW_KLUDGE_USE_DEFAULT_XMLNS=false
S3_GW_KLUDGE_BYPASS_CONTENT_ENCODING_CHECK_IN_CHUNKS=false
# Namespaces that should be handled as default
S3_GW_KLUDGE_DEFAULT_NAMESPACES="" "root"
# Enable bucket/object ACL support for newly created buckets.
S3_GW_KLUDGE_ACL_ENABLED=false
S3_GW_TRACING_ENABLED=false
S3_GW_TRACING_ENDPOINT="localhost:4318"
@ -203,8 +205,6 @@ S3_GW_FROSTFSID_CONTRACT=frostfsid.frostfs
S3_GW_FROSTFSID_VALIDATION_ENABLED=true
# Policy contract configuration. To enable this functionality the `rpc_endpoint` param must be also set.
# Enables using policies from Policy contract.
S3_GW_POLICY_ENABLED=true
# Policy contract hash (LE) or name in NNS.
S3_GW_POLICY_CONTRACT=policy.frostfs

View file

@ -193,6 +193,8 @@ kludge:
bypass_content_encoding_check_in_chunks: false
# Namespaces that should be handled as default
default_namespaces: [ "", "root" ]
# Enable bucket/object ACL support for newly created buckets.
acl_enabled: false
runtime:
soft_memory_limit: 1gb
@ -241,8 +243,6 @@ frostfsid:
# Policy contract configuration. To enable this functionality the `rpc_endpoint` param must be also set.
policy:
# Enables using policies from Policy contract.
enabled: true
# Policy contract hash (LE) or name in NNS.
contract: policy.frostfs

View file

@ -597,13 +597,15 @@ kludge:
use_default_xmlns: false
bypass_content_encoding_check_in_chunks: false
default_namespaces: [ "", "root" ]
acl_enabled: false
```
| Parameter | Type | SIGHUP reload | Default value | Description |
|-------------------------------------------|------------|---------------|---------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
| `use_default_xmlns` | `bool` | yes | false | Enable using default xml namespace `http://s3.amazonaws.com/doc/2006-03-01/` when parse xml bodies. |
| `bypass_content_encoding_check_in_chunks` | `bool` | yes | false | Use this flag to be able to use [chunked upload approach](https://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-streaming.html) without having `aws-chunked` value in `Content-Encoding` header. |
| `default_namespaces` | `[]string` | n/d | ["","root"] | Namespaces that should be handled as default. |
| `use_default_xmlns` | `bool` | yes | `false` | Enable using default xml namespace `http://s3.amazonaws.com/doc/2006-03-01/` when parse xml bodies. |
| `bypass_content_encoding_check_in_chunks` | `bool` | yes | `false` | Use this flag to be able to use [chunked upload approach](https://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-streaming.html) without having `aws-chunked` value in `Content-Encoding` header. |
| `default_namespaces` | `[]string` | yes | `["","root"]` | Namespaces that should be handled as default. |
| `acl_enabled` | `bool` | yes | `false` | Enable bucket/object ACL support for newly created buckets. |
# `runtime` section
Contains runtime parameters.
@ -673,14 +675,12 @@ Policy contract configuration. To enable this functionality the `rpc_endpoint` p
```yaml
policy:
enabled: false
contract: policy.frostfs
```
| Parameter | Type | SIGHUP reload | Default value | Description |
|------------|----------|---------------|----------------|-------------------------------------------------------------------|
| `enabled` | `bool` | no | true | Enables using policies from Policy contract to check permissions. |
| `contract` | `string` | no | policy.frostfs | Policy contract hash (LE) or name in NNS. |
| Parameter | Type | SIGHUP reload | Default value | Description |
|------------|----------|---------------|----------------|-------------------------------------------|
| `contract` | `string` | no | policy.frostfs | Policy contract hash (LE) or name in NNS. |
# `proxy` section

2
go.mod
View file

@ -7,7 +7,7 @@ require (
git.frostfs.info/TrueCloudLab/frostfs-contract v0.18.1-0.20231218084346-bce7ef18c83b
git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20230531082742-c97d21411eb6
git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20231107114540-ab75edd70939
git.frostfs.info/TrueCloudLab/policy-engine v0.0.0-20240206111236-8354a074c4df
git.frostfs.info/TrueCloudLab/policy-engine v0.0.0-20240226094215-c960b1b08831
git.frostfs.info/TrueCloudLab/zapjournald v0.0.0-20240124114243-cb2e66427d02
github.com/aws/aws-sdk-go v1.44.6
github.com/bluele/gcache v0.0.2

4
go.sum
View file

@ -48,8 +48,8 @@ git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20231107114540-ab75edd70939
git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20231107114540-ab75edd70939/go.mod h1:t1akKcUH7iBrFHX8rSXScYMP17k2kYQXMbZooiL5Juw=
git.frostfs.info/TrueCloudLab/hrw v1.2.1 h1:ccBRK21rFvY5R1WotI6LNoPlizk7qSvdfD8lNIRudVc=
git.frostfs.info/TrueCloudLab/hrw v1.2.1/go.mod h1:C1Ygde2n843yTZEQ0FP69jYiuaYV0kriLvP4zm8JuvM=
git.frostfs.info/TrueCloudLab/policy-engine v0.0.0-20240206111236-8354a074c4df h1:FLk850Ti+aj9vdJTUPvtS4KDIpISze9vTNKV15WIbME=
git.frostfs.info/TrueCloudLab/policy-engine v0.0.0-20240206111236-8354a074c4df/go.mod h1:YVL7yFaT0QNSpA0z+RHudLvrLwT+lsFYGyBSVc1ustI=
git.frostfs.info/TrueCloudLab/policy-engine v0.0.0-20240226094215-c960b1b08831 h1:yK2iGQlg5kMmU47ZHor/g52mVS1xEgJSRQ4Olp76Cg8=
git.frostfs.info/TrueCloudLab/policy-engine v0.0.0-20240226094215-c960b1b08831/go.mod h1:YVL7yFaT0QNSpA0z+RHudLvrLwT+lsFYGyBSVc1ustI=
git.frostfs.info/TrueCloudLab/rfc6979 v0.4.0 h1:M2KR3iBj7WpY3hP10IevfIB9MURr4O9mwVfJ+SjT3HA=
git.frostfs.info/TrueCloudLab/rfc6979 v0.4.0/go.mod h1:okpbKfVYf/BpejtfFTfhZqFP+sZ8rsHrP8Rr/jYPNRc=
git.frostfs.info/TrueCloudLab/tzhash v1.8.0 h1:UFMnUIk0Zh17m8rjGHJMqku2hCgaXDqjqZzS4gsb4UA=

View file

@ -14,7 +14,6 @@ import (
errorsFrost "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/frostfs/errors"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/acl"
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/eacl"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
@ -101,16 +100,8 @@ func (x *FrostFS) Container(ctx context.Context, idCnr cid.ID) (*container.Conta
return &res, nil
}
var basicACLZero acl.Basic
// CreateContainer implements frostfs.FrostFS interface method.
//
// If prm.BasicACL is zero, 'eacl-public-read-write' is used.
func (x *FrostFS) CreateContainer(ctx context.Context, prm layer.PrmContainerCreate) (*layer.ContainerCreateResult, error) {
if prm.BasicACL == basicACLZero {
prm.BasicACL = acl.PublicRWExtended
}
var cnr container.Container
cnr.Init()
cnr.SetPlacementPolicy(prm.Policy)

View file

@ -2,6 +2,7 @@ package frostfsid
import (
"context"
"encoding/hex"
"fmt"
"strconv"
"strings"
@ -100,6 +101,15 @@ func (f *FrostFSID) GetUserAddress(namespace, name string) (string, error) {
return key.Address(), nil
}
func (f *FrostFSID) GetUserKey(account, name string) (string, error) {
key, err := f.cli.GetSubjectKeyByName(account, name)
if err != nil {
return "", err
}
return hex.EncodeToString(key.Bytes()), nil
}
func (f *FrostFSID) GetUserGroupIDs(userHash util.Uint160) ([]string, error) {
subjExt, err := f.cli.GetSubjectExtended(userHash)
if err != nil {

View file

@ -2,9 +2,11 @@ package contract
import (
"context"
"errors"
"fmt"
"math/big"
"git.frostfs.info/TrueCloudLab/frostfs-contract/commonclient"
policycontract "git.frostfs.info/TrueCloudLab/frostfs-contract/policy"
policyclient "git.frostfs.info/TrueCloudLab/frostfs-contract/rpcclient/policy"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/frostfs/policy"
@ -21,6 +23,7 @@ import (
type Client struct {
actor *actor.Actor
policyContract *policyclient.Contract
contractHash util.Uint160
}
type Config struct {
@ -72,6 +75,7 @@ func New(ctx context.Context, cfg Config) (*Client, error) {
return &Client{
actor: act,
policyContract: policyclient.New(act, contractHash),
contractHash: contractHash,
}, nil
}
@ -130,3 +134,87 @@ func (c *Client) Wait(tx util.Uint256, vub uint32, err error) error {
_, err = c.actor.Wait(tx, vub, err)
return err
}
type multiTX struct {
contractHash util.Uint160
txs []*commonclient.Transaction
err error
}
func (m *multiTX) AddChain(entity policycontract.Kind, entityName string, name []byte, chain []byte) {
m.wrapCall("addChain", []any{big.NewInt(int64(entity)), entityName, name, chain})
}
func (m *multiTX) RemoveChain(entity policycontract.Kind, entityName string, name []byte) {
m.wrapCall("removeChain", []any{big.NewInt(int64(entity)), entityName, name})
}
func (m *multiTX) Scripts() ([][]byte, error) {
if m.err != nil {
return nil, m.err
}
if len(m.txs) == 0 {
return nil, errors.New("tx isn't initialized")
}
res := make([][]byte, 0, len(m.txs))
for _, tx := range m.txs {
script, err := tx.Bytes()
if err != nil {
return nil, err
}
res = append(res, script)
}
return res, nil
}
func (m *multiTX) wrapCall(method string, args []any) {
if m.err != nil {
return
}
if len(m.txs) == 0 {
m.err = errors.New("multi tx isn't initialized")
return
}
err := m.txs[len(m.txs)-1].WrapCall(method, args)
if err == nil {
return
}
if !errors.Is(commonclient.ErrTransactionTooLarge, err) {
m.err = err
return
}
tx := commonclient.NewTransaction(m.contractHash)
m.err = tx.WrapCall(method, args)
if m.err == nil {
m.txs = append(m.txs, tx)
}
}
func (c *Client) StartTx() policy.MultiTransaction {
return &multiTX{
txs: []*commonclient.Transaction{commonclient.NewTransaction(c.contractHash)},
contractHash: c.contractHash,
}
}
func (c *Client) SendTx(mtx policy.MultiTransaction) error {
var err error
scripts, err := mtx.Scripts()
if err != nil {
return err
}
for i := range scripts {
if _, err = c.actor.Wait(c.actor.SendRun(scripts[i])); err != nil {
return err
}
}
return nil
}

View file

@ -1,97 +0,0 @@
package contract
import (
"errors"
"strings"
"sync"
policycontract "git.frostfs.info/TrueCloudLab/frostfs-contract/policy"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/frostfs/policy"
"github.com/nspcc-dev/neo-go/pkg/util"
)
type InMemoryContract struct {
iamChains *syncedMap
containerChains *syncedMap
namespaceChains *syncedMap
}
type syncedMap struct {
mu sync.RWMutex
data map[string][]byte
}
var _ policy.Contract = (*InMemoryContract)(nil)
var ErrChainNotFound = errors.New("chain not found")
// NewInMemoryContract creates new inmemory Policy contract wrapper.
func NewInMemoryContract() *InMemoryContract {
return &InMemoryContract{
iamChains: &syncedMap{data: map[string][]byte{}},
containerChains: &syncedMap{data: map[string][]byte{}},
namespaceChains: &syncedMap{data: map[string][]byte{}},
}
}
func (c *InMemoryContract) AddChain(kind policycontract.Kind, entity string, name []byte, chain []byte) (util.Uint256, uint32, error) {
syncMap := c.getMap(kind)
syncMap.mu.Lock()
syncMap.data[entity+string(name)] = chain
syncMap.mu.Unlock()
return util.Uint256{}, 0, nil
}
func (c *InMemoryContract) GetChain(kind policycontract.Kind, entity string, name []byte) ([]byte, error) {
syncMap := c.getMap(kind)
syncMap.mu.RLock()
defer syncMap.mu.RUnlock()
val, ok := syncMap.data[entity+string(name)]
if !ok {
return nil, ErrChainNotFound
}
return val, nil
}
func (c *InMemoryContract) RemoveChain(kind policycontract.Kind, entity string, name []byte) (util.Uint256, uint32, error) {
syncMap := c.getMap(kind)
syncMap.mu.Lock()
delete(syncMap.data, entity+string(name))
syncMap.mu.Unlock()
return util.Uint256{}, 0, nil
}
func (c *InMemoryContract) ListChains(kind policycontract.Kind, entity string, name []byte) ([][]byte, error) {
syncMap := c.getMap(kind)
syncMap.mu.RLock()
defer syncMap.mu.RUnlock()
var res [][]byte
for key, val := range syncMap.data {
if strings.HasPrefix(key, entity+string(name)) {
res = append(res, val)
}
}
return res, nil
}
func (c *InMemoryContract) Wait(_ util.Uint256, _ uint32, err error) error {
return err
}
func (c *InMemoryContract) getMap(kind policycontract.Kind) *syncedMap {
switch kind {
case policycontract.IAM:
return c.iamChains
case policycontract.Container:
return c.containerChains
case policycontract.Namespace:
return c.namespaceChains
default:
return &syncedMap{data: map[string][]byte{}}
}
}

View file

@ -1,46 +0,0 @@
package policy
import (
policycontract "git.frostfs.info/TrueCloudLab/frostfs-contract/policy"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/handler"
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
"go.uber.org/zap"
)
type MorphPolicyStorage struct {
contract Contract
}
type MorphPolicyStorageConfig struct {
Contract Contract
Log *zap.Logger
}
var _ handler.PolicyStorage = (*MorphPolicyStorage)(nil)
const policyStoragePrefix = 'b'
func NewMorphPolicyStorage(config *MorphPolicyStorageConfig) *MorphPolicyStorage {
return &MorphPolicyStorage{
contract: config.Contract,
}
}
func (c *MorphPolicyStorage) PutPolicy(namespace string, cnrID cid.ID, policy []byte) error {
name := getPolicyStorageName(cnrID)
return c.contract.Wait(c.contract.AddChain(policycontract.IAM, namespace, name, policy))
}
func (c *MorphPolicyStorage) GetPolicy(namespace string, cnrID cid.ID) ([]byte, error) {
name := getPolicyStorageName(cnrID)
return c.contract.GetChain(policycontract.IAM, namespace, name)
}
func (c *MorphPolicyStorage) DeletePolicy(namespace string, cnrID cid.ID) error {
name := getPolicyStorageName(cnrID)
return c.contract.Wait(c.contract.RemoveChain(policycontract.IAM, namespace, name))
}
func getPolicyStorageName(cnrID cid.ID) []byte {
return append([]byte{policyStoragePrefix}, cnrID[:]...)
}

View file

@ -5,8 +5,8 @@ import (
policycontract "git.frostfs.info/TrueCloudLab/frostfs-contract/policy"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/cache"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/handler"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs"
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
"git.frostfs.info/TrueCloudLab/policy-engine/pkg/chain"
"git.frostfs.info/TrueCloudLab/policy-engine/pkg/engine"
"github.com/nspcc-dev/neo-go/pkg/util"
@ -25,10 +25,9 @@ type MorphRuleChainStorageConfig struct {
Log *zap.Logger
}
var (
_ engine.MorphRuleChainStorage = (*MorphRuleChainStorage)(nil)
_ handler.MorphRuleChainStorage = (*MorphRuleChainStorage)(nil)
)
var _ engine.MorphRuleChainStorage = (*MorphRuleChainStorage)(nil)
const bucketPolicyPrefix = 'b'
func NewMorphRuleChainStorage(config *MorphRuleChainStorageConfig) *MorphRuleChainStorage {
return &MorphRuleChainStorage{
@ -38,26 +37,12 @@ func NewMorphRuleChainStorage(config *MorphRuleChainStorageConfig) *MorphRuleCha
}
}
func (c *MorphRuleChainStorage) AddChain(target engine.Target, policyChain *chain.Chain) error {
return c.contract.Wait(c.AddMorphRuleChain(chain.S3, target, policyChain))
func (c *MorphRuleChainStorage) AddMorphRuleChain(chain.Name, engine.Target, *chain.Chain) (util.Uint256, uint32, error) {
panic("should never be called")
}
func (c *MorphRuleChainStorage) RemoveChain(target engine.Target, chainID chain.ID) error {
return c.contract.Wait(c.RemoveMorphRuleChain(chain.S3, target, chainID))
}
func (c *MorphRuleChainStorage) ListChains(target engine.Target) ([]*chain.Chain, error) {
return c.ListMorphRuleChains(chain.S3, target)
}
func (c *MorphRuleChainStorage) AddMorphRuleChain(name chain.Name, target engine.Target, policyChain *chain.Chain) (util.Uint256, uint32, error) {
c.cache.Delete(cache.MorphPolicyCacheKey{Target: target, Name: name})
return c.contract.AddChain(getKind(target), target.Name, getName(name, policyChain.ID), policyChain.Bytes())
}
func (c *MorphRuleChainStorage) RemoveMorphRuleChain(name chain.Name, target engine.Target, chainID chain.ID) (util.Uint256, uint32, error) {
c.cache.Delete(cache.MorphPolicyCacheKey{Target: target, Name: name})
return c.contract.RemoveChain(getKind(target), target.Name, getName(name, chainID))
func (c *MorphRuleChainStorage) RemoveMorphRuleChain(chain.Name, engine.Target, chain.ID) (util.Uint256, uint32, error) {
panic("should never be called")
}
func (c *MorphRuleChainStorage) ListMorphRuleChains(name chain.Name, target engine.Target) ([]*chain.Chain, error) {
@ -88,6 +73,45 @@ func (c *MorphRuleChainStorage) ListMorphRuleChains(name chain.Name, target engi
return list, nil
}
func (c *MorphRuleChainStorage) PutBucketPolicy(ns string, cnrID cid.ID, policy []byte, chains []*chain.Chain) error {
c.cache.Delete(cache.MorphPolicyCacheKey{Target: engine.NamespaceTarget(ns), Name: chain.S3})
tx := c.contract.StartTx()
tx.AddChain(policycontract.IAM, ns, getBucketPolicyName(cnrID), policy)
for i := range chains {
tx.AddChain(policycontract.Namespace, ns, chains[i].ID, chains[i].Bytes())
}
return c.contract.SendTx(tx)
}
func (c *MorphRuleChainStorage) DeleteBucketPolicy(ns string, cnrID cid.ID, chainID chain.ID) error {
c.cache.Delete(cache.MorphPolicyCacheKey{Target: engine.NamespaceTarget(ns), Name: chain.S3})
tx := c.contract.StartTx()
tx.RemoveChain(policycontract.Namespace, ns, chainID)
tx.RemoveChain(policycontract.IAM, ns, getBucketPolicyName(cnrID))
return c.contract.SendTx(tx)
}
func (c *MorphRuleChainStorage) GetBucketPolicy(ns string, cnrID cid.ID) ([]byte, error) {
return c.contract.GetChain(policycontract.IAM, ns, getBucketPolicyName(cnrID))
}
func (c *MorphRuleChainStorage) SaveACLChains(ns string, chains []*chain.Chain) error {
c.cache.Delete(cache.MorphPolicyCacheKey{Target: engine.NamespaceTarget(ns), Name: chain.S3})
c.cache.Delete(cache.MorphPolicyCacheKey{Target: engine.NamespaceTarget(ns), Name: chain.Ingress})
tx := c.contract.StartTx()
for i := range chains {
tx.AddChain(policycontract.Namespace, ns, chains[i].ID, chains[i].Bytes())
}
return c.contract.SendTx(tx)
}
func getKind(target engine.Target) policycontract.Kind {
var kind policycontract.Kind = policycontract.Container
if target.Type != engine.Container {
@ -96,6 +120,7 @@ func getKind(target engine.Target) policycontract.Kind {
return kind
}
func getName(name chain.Name, chainID chain.ID) []byte {
return append([]byte(name), []byte(chainID)...)
func getBucketPolicyName(cnrID cid.ID) []byte {
return append([]byte{bucketPolicyPrefix}, cnrID[:]...)
}

View file

@ -9,18 +9,15 @@ import (
"git.frostfs.info/TrueCloudLab/policy-engine/pkg/engine"
"git.frostfs.info/TrueCloudLab/policy-engine/pkg/engine/inmemory"
"git.frostfs.info/TrueCloudLab/policy-engine/pkg/resource"
"github.com/nspcc-dev/neo-go/pkg/util"
"go.uber.org/zap"
)
type Storage struct {
router engine.ChainRouter
morph handler.MorphRuleChainStorage
morph *MorphRuleChainStorage
local engine.LocalOverrideStorage
policy handler.PolicyStorage
}
type StorageConfig struct {
@ -29,18 +26,23 @@ type StorageConfig struct {
Log *zap.Logger
}
type MultiTransaction interface {
AddChain(entity policycontract.Kind, entityName string, name []byte, chain []byte)
RemoveChain(entity policycontract.Kind, entityName string, name []byte)
Scripts() ([][]byte, error)
}
type Contract interface {
AddChain(kind policycontract.Kind, entity string, name []byte, chain []byte) (util.Uint256, uint32, error)
GetChain(kind policycontract.Kind, entity string, name []byte) ([]byte, error)
RemoveChain(kind policycontract.Kind, entity string, name []byte) (util.Uint256, uint32, error)
ListChains(kind policycontract.Kind, entity string, name []byte) ([][]byte, error)
Wait(tx util.Uint256, vub uint32, err error) error
GetChain(entity policycontract.Kind, entityName string, name []byte) ([]byte, error)
ListChains(entity policycontract.Kind, entityName string, prefix []byte) ([][]byte, error)
StartTx() MultiTransaction
SendTx(transaction MultiTransaction) error
}
var _ handler.APE = (*Storage)(nil)
func NewStorage(cfg StorageConfig) *Storage {
// todo use thread safe inmemory https://git.frostfs.info/TrueCloudLab/policy-engine/issues/35
local := inmemory.NewInmemoryLocalStorage()
morph := NewMorphRuleChainStorage(&MorphRuleChainStorageConfig{
@ -49,16 +51,10 @@ func NewStorage(cfg StorageConfig) *Storage {
Log: cfg.Log,
})
policyStorage := NewMorphPolicyStorage(&MorphPolicyStorageConfig{
Contract: cfg.Contract,
Log: cfg.Log,
})
return &Storage{
router: engine.NewDefaultChainRouterWithLocalOverrides(morph, local),
morph: morph,
local: local,
policy: policyStorage,
}
}
@ -70,26 +66,18 @@ func (s *Storage) LocalStorage() engine.LocalOverrideStorage {
return s.local
}
func (s *Storage) AddChain(target engine.Target, policyChain *chain.Chain) error {
return s.morph.AddChain(target, policyChain)
func (s *Storage) PutBucketPolicy(ns string, cnrID cid.ID, policy []byte, policyChains []*chain.Chain) error {
return s.morph.PutBucketPolicy(ns, cnrID, policy, policyChains)
}
func (s *Storage) RemoveChain(target engine.Target, chainID chain.ID) error {
return s.morph.RemoveChain(target, chainID)
func (s *Storage) DeleteBucketPolicy(ns string, cnrID cid.ID, chainID chain.ID) error {
return s.morph.DeleteBucketPolicy(ns, cnrID, chainID)
}
func (s *Storage) ListChains(target engine.Target) ([]*chain.Chain, error) {
return s.morph.ListChains(target)
func (s *Storage) GetBucketPolicy(ns string, cnrID cid.ID) ([]byte, error) {
return s.morph.GetBucketPolicy(ns, cnrID)
}
func (s *Storage) PutPolicy(namespace string, cnrID cid.ID, policy []byte) error {
return s.policy.PutPolicy(namespace, cnrID, policy)
}
func (s *Storage) GetPolicy(namespace string, cnrID cid.ID) ([]byte, error) {
return s.policy.GetPolicy(namespace, cnrID)
}
func (s *Storage) DeletePolicy(namespace string, cnrID cid.ID) error {
return s.policy.DeletePolicy(namespace, cnrID)
func (s *Storage) SaveACLChains(ns string, chains []*chain.Chain) error {
return s.morph.SaveACLChains(ns, chains)
}

View file

@ -141,4 +141,9 @@ const (
CouldntDeleteObjectFromStorageContinueDeleting = "couldn't delete object from storage, continue deleting from tree"
CouldntPutAccessBoxIntoCache = "couldn't put accessbox into cache"
InvalidAccessBoxCacheRemovingCheckInterval = "invalid accessbox check removing interval, using default value"
CouldNotParseContainerAPEEnabledAttribute = "could not parse container APE enabled attribute"
CouldNotCloseRequestBody = "could not close request body"
BucketOwnerKeyIsMissing = "bucket owner key is missing"
SettingsNodeInvalidOwnerKey = "settings node: invalid owner key"
FailedToSendTransaction = "failed to send transaction"
)

View file

@ -2,6 +2,7 @@ package tree
import (
"context"
"encoding/hex"
"errors"
"fmt"
"io"
@ -16,6 +17,7 @@ import (
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs"
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user"
"github.com/nspcc-dev/neo-go/pkg/crypto/keys"
"go.uber.org/zap"
"golang.org/x/exp/maps"
)
@ -78,6 +80,8 @@ var (
const (
versioningKV = "Versioning"
cannedACLKV = "cannedACL"
ownerKeyKV = "ownerKey"
lockConfigurationKV = "LockConfiguration"
oidKV = "OID"
@ -332,7 +336,7 @@ func newPartInfo(node NodeResponse) (*data.PartInfo, error) {
}
func (c *Tree) GetSettingsNode(ctx context.Context, bktInfo *data.BucketInfo) (*data.BucketSettings, error) {
keysToReturn := []string{versioningKV, lockConfigurationKV}
keysToReturn := []string{versioningKV, lockConfigurationKV, cannedACLKV, ownerKeyKV}
node, err := c.getSystemNode(ctx, bktInfo, []string{settingsFileName}, keysToReturn)
if err != nil {
return nil, fmt.Errorf("couldn't get node: %w", err)
@ -349,6 +353,14 @@ func (c *Tree) GetSettingsNode(ctx context.Context, bktInfo *data.BucketInfo) (*
}
}
settings.CannedACL, _ = node.Get(cannedACLKV)
if ownerKeyHex, ok := node.Get(ownerKeyKV); ok {
if settings.OwnerKey, err = keys.NewPublicKeyFromString(ownerKeyHex); err != nil {
c.reqLogger(ctx).Error(logs.SettingsNodeInvalidOwnerKey, zap.Error(err))
}
}
return settings, nil
}
@ -1384,6 +1396,10 @@ func metaFromSettings(settings *data.BucketSettings) map[string]string {
results[FileNameKey] = settingsFileName
results[versioningKV] = settings.Versioning
results[lockConfigurationKV] = encodeLockConfiguration(settings.LockConfiguration)
results[cannedACLKV] = settings.CannedACL
if settings.OwnerKey != nil {
results[ownerKeyKV] = hex.EncodeToString(settings.OwnerKey.Bytes())
}
return results
}

View file

@ -9,6 +9,7 @@ import (
cidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id/test"
oidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id/test"
usertest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user/test"
"github.com/nspcc-dev/neo-go/pkg/crypto/keys"
"github.com/stretchr/testify/require"
"go.uber.org/zap/zaptest"
)
@ -111,6 +112,9 @@ func TestTreeServiceSettings(t *testing.T) {
CID: cidtest.ID(),
}
key, err := keys.NewPrivateKey()
require.NoError(t, err)
settings := &data.BucketSettings{
Versioning: "Versioning",
LockConfiguration: &data.ObjectLockConfiguration{
@ -122,6 +126,7 @@ func TestTreeServiceSettings(t *testing.T) {
},
},
},
OwnerKey: key.PublicKey(),
}
err = treeService.PutSettingsNode(ctx, bktInfo, settings)