From a7ce40d7450af145efa92effe2674cdb7f3d94e9 Mon Sep 17 00:00:00 2001 From: Marina Biryukova Date: Thu, 3 Apr 2025 13:51:16 +0300 Subject: [PATCH] [#535] Support public access block operations Signed-off-by: Marina Biryukova --- api/cache/policy.go | 33 +- api/data/info.go | 9 + api/errors/errors.go | 7 + api/handler/access_block.go | 217 ++++++++++++ api/handler/access_block_test.go | 311 ++++++++++++++++++ api/handler/acl.go | 52 ++- api/handler/acl_test.go | 5 + api/handler/api.go | 1 + api/handler/handlers_test.go | 14 + api/handler/locking.go | 11 +- api/middleware/auth.go | 70 +++- api/middleware/constants.go | 60 ++-- api/middleware/policy.go | 6 + api/middleware/policy_test.go | 18 + api/middleware/reqinfo.go | 3 +- api/router.go | 27 +- api/router_mock_test.go | 44 ++- api/router_test.go | 37 ++- internal/frostfs/frostfsid/frostfsid.go | 10 +- .../policy/morph_rule_chain_storage.go | 40 ++- internal/frostfs/policy/storage.go | 4 + internal/logs/logs.go | 1 + pkg/service/tree/tree.go | 47 +++ 23 files changed, 940 insertions(+), 87 deletions(-) create mode 100644 api/handler/access_block.go create mode 100644 api/handler/access_block_test.go diff --git a/api/cache/policy.go b/api/cache/policy.go index 9d05843bc..9b39ae429 100644 --- a/api/cache/policy.go +++ b/api/cache/policy.go @@ -44,7 +44,7 @@ func NewMorphPolicyCache(config *Config) *MorphPolicyCache { return &MorphPolicyCache{cache: gc, logger: config.Logger} } -// Get returns a cached object. Returns nil if value is missing. +// Get returns cached chains. Returns nil if value is missing. func (o *MorphPolicyCache) Get(key MorphPolicyCacheKey) []*chain.Chain { entry, err := o.cache.Get(key) if err != nil { @@ -61,12 +61,39 @@ func (o *MorphPolicyCache) Get(key MorphPolicyCacheKey) []*chain.Chain { return result } -// Put puts an object to cache. +// Put puts chains to cache. func (o *MorphPolicyCache) Put(key MorphPolicyCacheKey, list []*chain.Chain) error { return o.cache.Set(key, list) } -// Delete deletes an object from cache. +// Delete deletes chains from cache. func (o *MorphPolicyCache) Delete(key MorphPolicyCacheKey) bool { return o.cache.Remove(key) } + +// GetBucketPolicy returns cached bucket policy. Returns nil if value is missing. +func (o *MorphPolicyCache) GetBucketPolicy(name string) []byte { + entry, err := o.cache.Get(name) + if err != nil { + return nil + } + + result, ok := entry.([]byte) + if !ok { + o.logger.Warn(logs.InvalidCacheEntryType, zap.String("actual", fmt.Sprintf("%T", entry)), + zap.String("expected", fmt.Sprintf("%T", result)), logs.TagField(logs.TagDatapath)) + return nil + } + + return result +} + +// PutBucketPolicy puts bucket policy to cache. +func (o *MorphPolicyCache) PutBucketPolicy(name string, policy []byte) error { + return o.cache.Set(name, policy) +} + +// DeleteBucketPolicy deletes bucket policy from cache. +func (o *MorphPolicyCache) DeleteBucketPolicy(name string) bool { + return o.cache.Remove(name) +} diff --git a/api/data/info.go b/api/data/info.go index adf77d3b8..e06a1310c 100644 --- a/api/data/info.go +++ b/api/data/info.go @@ -65,6 +65,7 @@ type ( LockConfiguration *ObjectLockConfiguration CannedACL string OwnerKey *keys.PublicKey + PublicAccessBlock *PublicAccessBlockConfiguration } // Versioning stores bucket versioning settings. @@ -106,6 +107,14 @@ type ( MD5Sum []byte CreationEpoch uint64 } + + PublicAccessBlockConfiguration struct { + XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ PublicAccessBlockConfiguration" json:"-"` + BlockPublicAcls bool `xml:"BlockPublicAcls" json:"BlockPublicAcls"` + IgnorePublicAcls bool `xml:"IgnorePublicAcls" json:"IgnorePublicAcls"` + BlockPublicPolicy bool `xml:"BlockPublicPolicy" json:"BlockPublicPolicy"` + RestrictPublicBuckets bool `xml:"RestrictPublicBuckets" json:"RestrictPublicBuckets"` + } ) // SettingsObjectName is a system name for a bucket settings file. diff --git a/api/errors/errors.go b/api/errors/errors.go index 2a17d544a..9e4005424 100644 --- a/api/errors/errors.go +++ b/api/errors/errors.go @@ -199,6 +199,7 @@ const ( ErrRangeOutOfBounds ErrMissingContentRange ErrNoSuchTagSet + ErrNoSuchPublicAccessBlockConfiguration ErrMalformedJSON ErrInsecureClientRequest @@ -1830,6 +1831,12 @@ var errorCodes = errorCodeMap{ Description: "The Versioning element must be specified", HTTPStatusCode: http.StatusBadRequest, }, + ErrNoSuchPublicAccessBlockConfiguration: { + ErrCode: ErrNoSuchPublicAccessBlockConfiguration, + Code: "NoSuchPublicAccessBlockConfiguration", + Description: "The public access block configuration was not found", + HTTPStatusCode: http.StatusNotFound, + }, // Add your error structure here. } diff --git a/api/handler/access_block.go b/api/handler/access_block.go new file mode 100644 index 000000000..763784c06 --- /dev/null +++ b/api/handler/access_block.go @@ -0,0 +1,217 @@ +package handler + +import ( + "bytes" + "context" + "encoding/base64" + "encoding/json" + "fmt" + "io" + "net/http" + "strings" + + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" + "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api" + "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data" + apierr "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors" + "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer" + "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/middleware" + s3common "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/pkg/policy-engine/common" + "git.frostfs.info/TrueCloudLab/policy-engine/pkg/chain" + "go.uber.org/zap" +) + +func (h *handler) PutPublicAccessBlockHandler(w http.ResponseWriter, r *http.Request) { + ctx, span := tracing.StartSpanFromContext(r.Context(), "handler.PutPublicAccessBlock") + defer span.End() + + var buf bytes.Buffer + + tee := io.TeeReader(r.Body, &buf) + reqInfo := middleware.GetReqInfo(ctx) + + cfg := new(data.PublicAccessBlockConfiguration) + if err := h.cfg.NewXMLDecoder(tee, r.UserAgent()).Decode(cfg); err != nil { + h.logAndSendError(ctx, w, "could not decode body", reqInfo, fmt.Errorf("%w: %s", apierr.GetAPIError(apierr.ErrMalformedXML), err.Error())) + return + } + + if _, ok := r.Header[api.ContentMD5]; ok { + headerMD5, err := base64.StdEncoding.DecodeString(r.Header.Get(api.ContentMD5)) + if err != nil { + h.logAndSendError(ctx, w, "invalid Content-MD5", reqInfo, apierr.GetAPIError(apierr.ErrInvalidDigest)) + return + } + + bodyMD5, err := getContentMD5(&buf) + if err != nil { + h.logAndSendError(ctx, w, "could not get content md5", reqInfo, err) + return + } + + if !bytes.Equal(headerMD5, bodyMD5) { + h.logAndSendError(ctx, w, "Content-MD5 does not match", reqInfo, apierr.GetAPIError(apierr.ErrInvalidDigest)) + return + } + } + + bktInfo, err := h.getBucketAndCheckOwner(r, reqInfo.BucketName) + if err != nil { + h.logAndSendError(ctx, w, "could not get bucket info", reqInfo, err) + return + } + + settings, err := h.obj.GetBucketSettings(ctx, bktInfo) + if err != nil { + h.logAndSendError(ctx, w, "couldn't get bucket settings", reqInfo, err) + return + } + + // Delete ACL chains if IgnorePublicAcls is set to true + if (settings.PublicAccessBlock == nil || !settings.PublicAccessBlock.IgnorePublicAcls) && cfg.IgnorePublicAcls && settings.CannedACL != basicACLPrivate { + if err = h.policyEngine.APE.DeleteACLChains(bktInfo.CID.EncodeToString(), []chain.ID{ + getBucketCannedChainID(chain.S3, bktInfo.CID), + getBucketCannedChainID(chain.Ingress, bktInfo.CID), + }); err != nil { + h.logAndSendError(ctx, w, "failed to delete morph rule chains", reqInfo, err) + return + } + } + + // Set ACL chains if IgnorePublicAcls is set to false + if settings.PublicAccessBlock != nil && settings.PublicAccessBlock.IgnorePublicAcls && !cfg.IgnorePublicAcls { + chainRules := bucketCannedACLToAPERules(settings.CannedACL, reqInfo, bktInfo.CID) + if err = h.policyEngine.APE.SaveACLChains(bktInfo.CID.EncodeToString(), chainRules); err != nil { + h.logAndSendError(ctx, w, "failed to add morph rule chains", reqInfo, err) + return + } + } + + newSettings := *settings + newSettings.PublicAccessBlock = cfg + sp := &layer.PutSettingsParams{ + BktInfo: bktInfo, + Settings: &newSettings, + } + + if err = h.obj.PutBucketSettings(ctx, sp); err != nil { + h.logAndSendError(ctx, w, "couldn't save bucket settings", reqInfo, err, + zap.String("container_id", bktInfo.CID.EncodeToString())) + return + } + + w.WriteHeader(http.StatusOK) +} + +func (h *handler) GetPublicAccessBlockHandler(w http.ResponseWriter, r *http.Request) { + ctx, span := tracing.StartSpanFromContext(r.Context(), "handler.GetPublicAccessBlock") + defer span.End() + + reqInfo := middleware.GetReqInfo(ctx) + + bktInfo, err := h.getBucketAndCheckOwner(r, reqInfo.BucketName) + if err != nil { + h.logAndSendError(ctx, w, "could not get bucket info", reqInfo, err) + return + } + + settings, err := h.obj.GetBucketSettings(ctx, bktInfo) + if err != nil { + h.logAndSendError(ctx, w, "couldn't get bucket settings", reqInfo, err) + return + } + + if settings.PublicAccessBlock == nil { + h.logAndSendError(ctx, w, "no public access block", reqInfo, apierr.GetAPIError(apierr.ErrNoSuchPublicAccessBlockConfiguration)) + return + } + + if err = middleware.EncodeToResponse(w, settings.PublicAccessBlock); err != nil { + h.logAndSendError(ctx, w, "something went wrong", reqInfo, err) + return + } +} + +func (h *handler) DeletePublicAccessBlockHandler(w http.ResponseWriter, r *http.Request) { + ctx, span := tracing.StartSpanFromContext(r.Context(), "handler.DeletePublicAccessBlock") + defer span.End() + + reqInfo := middleware.GetReqInfo(ctx) + + bktInfo, err := h.getBucketAndCheckOwner(r, reqInfo.BucketName) + if err != nil { + h.logAndSendError(ctx, w, "could not get bucket info", reqInfo, err) + return + } + + settings, err := h.obj.GetBucketSettings(ctx, bktInfo) + if err != nil { + h.logAndSendError(ctx, w, "couldn't get bucket settings", reqInfo, err) + return + } + + if settings.PublicAccessBlock == nil { + w.WriteHeader(http.StatusNoContent) + return + } + + // Set ACL chains if IgnorePublicAcls was set to true + if settings.PublicAccessBlock.IgnorePublicAcls { + chainRules := bucketCannedACLToAPERules(settings.CannedACL, reqInfo, bktInfo.CID) + if err = h.policyEngine.APE.SaveACLChains(bktInfo.CID.EncodeToString(), chainRules); err != nil { + h.logAndSendError(ctx, w, "failed to add morph rule chains", reqInfo, err) + return + } + } + + newSettings := *settings + newSettings.PublicAccessBlock = nil + sp := &layer.PutSettingsParams{ + BktInfo: bktInfo, + Settings: &newSettings, + } + if err = h.obj.PutBucketSettings(ctx, sp); err != nil { + h.logAndSendError(ctx, w, "couldn't save bucket settings", reqInfo, err, + zap.String("container_id", bktInfo.CID.EncodeToString())) + return + } + + w.WriteHeader(http.StatusNoContent) +} + +func (h *handler) CheckRestrictPublicBuckets(ctx context.Context) error { + reqInfo := middleware.GetReqInfo(ctx) + + bktInfo, err := h.obj.GetBucketInfo(ctx, reqInfo.BucketName) + if err != nil { + return fmt.Errorf("get bucket info: %w", err) + } + + settings, err := h.obj.GetBucketSettings(ctx, bktInfo) + if err != nil { + return fmt.Errorf("get bucket settings: %w", err) + } + + if settings.PublicAccessBlock != nil && settings.PublicAccessBlock.RestrictPublicBuckets { + jsonPolicy, err := h.policyEngine.APE.GetBucketPolicy(reqInfo.Namespace, bktInfo.CID) + if err != nil { + if strings.Contains(err.Error(), "not found") { + return nil + } + return fmt.Errorf("get bucket policy: %w", err) + } + + var bktPolicy s3common.Policy + if err = json.Unmarshal(jsonPolicy, &bktPolicy); err != nil { + return fmt.Errorf("unmarshal bucket policy: %w", err) + } + + // Check whether bucket policy is public and namespaces of bucket and user are equal + if getPolicyStatus(bktPolicy) == PolicyStatusIsPublicTrue && + (reqInfo.UserNamespace == nil || *reqInfo.UserNamespace != reqInfo.Namespace) { + return fmt.Errorf("public buckets are restricted: %w", apierr.GetAPIError(apierr.ErrAccessDenied)) + } + } + + return nil +} diff --git a/api/handler/access_block_test.go b/api/handler/access_block_test.go new file mode 100644 index 000000000..f6a4c1cc1 --- /dev/null +++ b/api/handler/access_block_test.go @@ -0,0 +1,311 @@ +package handler + +import ( + "bytes" + "context" + "encoding/base64" + "encoding/xml" + "fmt" + "net/http" + "net/http/httptest" + "testing" + + "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api" + "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data" + apierr "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors" + "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/middleware" + "git.frostfs.info/TrueCloudLab/policy-engine/iam" + "git.frostfs.info/TrueCloudLab/policy-engine/pkg/engine" + "github.com/nspcc-dev/neo-go/pkg/crypto/keys" + "github.com/stretchr/testify/require" +) + +func TestGetPublicAccessBlock(t *testing.T) { + hc := prepareHandlerContext(t) + bktName := "bucket-get-public-access-block" + + createBucket(hc, bktName) + deletePublicAccessBlock(hc, bktName) + getPublicAccessBlockErr(hc, bktName, apierr.GetAPIError(apierr.ErrNoSuchPublicAccessBlockConfiguration)) + + putPublicAccessBlock(hc, bktName, &data.PublicAccessBlockConfiguration{ + BlockPublicAcls: true, + BlockPublicPolicy: true, + IgnorePublicAcls: true, + RestrictPublicBuckets: true, + }, nil) + + cfg := getPublicAccessBlock(hc, bktName) + require.True(t, cfg.BlockPublicAcls) + require.True(t, cfg.BlockPublicPolicy) + require.True(t, cfg.IgnorePublicAcls) + require.True(t, cfg.RestrictPublicBuckets) + + deletePublicAccessBlock(hc, bktName) + getPublicAccessBlockErr(hc, bktName, apierr.GetAPIError(apierr.ErrNoSuchPublicAccessBlockConfiguration)) +} + +func TestPutPublicAccessBlock(t *testing.T) { + hc := prepareHandlerContext(t) + bktName := "bucket-put-public-access-block" + + createBucket(hc, bktName) + + cfg := &data.PublicAccessBlockConfiguration{ + BlockPublicAcls: true, + BlockPublicPolicy: true, + IgnorePublicAcls: true, + RestrictPublicBuckets: true, + } + body, err := xml.Marshal(cfg) + require.NoError(t, err) + contentMD5, err := getContentMD5(bytes.NewReader(body)) + require.NoError(t, err) + + putPublicAccessBlockErr(hc, bktName, cfg, map[string]string{api.AmzExpectedBucketOwner: "owner"}, apierr.GetAPIError(apierr.ErrAccessDenied)) + putPublicAccessBlockErr(hc, bktName, cfg, map[string]string{api.ContentMD5: "invalid"}, apierr.GetAPIError(apierr.ErrInvalidDigest)) + putPublicAccessBlockErr(hc, bktName, cfg, map[string]string{api.ContentMD5: base64.StdEncoding.EncodeToString([]byte{})}, apierr.GetAPIError(apierr.ErrInvalidDigest)) + putPublicAccessBlock(hc, bktName, cfg, map[string]string{api.ContentMD5: base64.StdEncoding.EncodeToString(contentMD5)}) + + w, r := prepareTestRequest(hc, bktName, "", &data.LifecycleConfiguration{}) + hc.Handler().PutPublicAccessBlockHandler(w, r) + assertS3Error(t, w, apierr.GetAPIError(apierr.ErrMalformedXML)) +} + +func TestBlockPublicAcls(t *testing.T) { + hc := prepareHandlerContext(t) + bktName := "bucket-block-public-acls" + + info := createBucket(hc, bktName) + putPublicAccessBlock(hc, bktName, &data.PublicAccessBlockConfiguration{ + BlockPublicAcls: true, + }, nil) + + putBucketACLErr(hc, bktName, info.Box, map[string]string{api.AmzACL: basicACLPublic}, apierr.GetAPIError(apierr.ErrAccessDenied)) + putBucketACLErr(hc, bktName, info.Box, map[string]string{api.AmzACL: basicACLReadOnly}, apierr.GetAPIError(apierr.ErrAccessDenied)) + putBucketACL(hc, bktName, info.Box, map[string]string{api.AmzACL: basicACLPrivate}) +} + +func TestBlockPublicPolicy(t *testing.T) { + hc := prepareHandlerContext(t) + bktName := "bucket-block-public-policy" + + createBucket(hc, bktName) + putPublicAccessBlock(hc, bktName, &data.PublicAccessBlockConfiguration{ + BlockPublicPolicy: true, + }, nil) + + putBucketPolicy(hc, bktName, iam.Policy{ // public policy + Version: "2012-10-17", + Statement: []iam.Statement{ + { + Principal: map[iam.PrincipalType][]string{iam.Wildcard: {}}, + Effect: "Allow", + Action: []string{"s3:*"}, + Resource: []string{fmt.Sprintf("arn:aws:s3:::%s/*", bktName)}, + }, + }, + }, apierr.ErrAccessDenied) + + key, err := keys.NewPrivateKey() + require.NoError(t, err) + hc.Handler().frostfsid.(*frostfsidMock).data["devenv"] = key.PublicKey() + putBucketPolicy(hc, bktName, iam.Policy{ // non-public policy + Version: "2012-10-17", + Statement: []iam.Statement{ + { + Principal: map[iam.PrincipalType][]string{iam.AWSPrincipalType: {"arn:aws:iam:::user/devenv"}}, + Effect: "Allow", + Action: []string{"s3:*"}, + Resource: []string{fmt.Sprintf("arn:aws:s3:::%s/*", bktName)}, + }, + }, + }) +} + +func TestIgnorePublicAcls(t *testing.T) { + hc := prepareHandlerContext(t) + bktName := "bucket-ignore-public-acls" + + info := createBucket(hc, bktName) + chains := hc.h.policyEngine.APE.(*apeMock).chainMap[engine.ContainerTarget(info.BktInfo.CID.EncodeToString())] + require.Len(t, chains, 2) + require.Len(t, chains[0].Rules, 0) + require.Len(t, chains[1].Rules, 0) + + putPublicAccessBlock(hc, bktName, &data.PublicAccessBlockConfiguration{ + IgnorePublicAcls: true, + }, nil) + chains = hc.h.policyEngine.APE.(*apeMock).chainMap[engine.ContainerTarget(info.BktInfo.CID.EncodeToString())] + require.Len(t, chains, 2) + require.Len(t, chains[0].Rules, 0) + require.Len(t, chains[1].Rules, 0) + + putBucketACL(hc, bktName, info.Box, map[string]string{api.AmzACL: basicACLPrivate}) + chains = hc.h.policyEngine.APE.(*apeMock).chainMap[engine.ContainerTarget(info.BktInfo.CID.EncodeToString())] + require.Len(t, chains, 2) + require.Len(t, chains[0].Rules, 0) + require.Len(t, chains[1].Rules, 0) + + putBucketACL(hc, bktName, info.Box, map[string]string{api.AmzACL: basicACLPublic}) + chains = hc.h.policyEngine.APE.(*apeMock).chainMap[engine.ContainerTarget(info.BktInfo.CID.EncodeToString())] + require.Len(t, chains, 2) + require.Len(t, chains[0].Rules, 0) + require.Len(t, chains[1].Rules, 0) + + putBucketACL(hc, bktName, info.Box, map[string]string{api.AmzACL: basicACLReadOnly}) + chains = hc.h.policyEngine.APE.(*apeMock).chainMap[engine.ContainerTarget(info.BktInfo.CID.EncodeToString())] + require.Len(t, chains, 2) + require.Len(t, chains[0].Rules, 0) + require.Len(t, chains[1].Rules, 0) + + putPublicAccessBlock(hc, bktName, &data.PublicAccessBlockConfiguration{ + IgnorePublicAcls: false, + }, nil) + chains = hc.h.policyEngine.APE.(*apeMock).chainMap[engine.ContainerTarget(info.BktInfo.CID.EncodeToString())] + require.Len(t, chains, 2) + require.Len(t, chains[0].Rules, 1) + require.Len(t, chains[1].Rules, 1) + + putBucketACL(hc, bktName, info.Box, map[string]string{api.AmzACL: basicACLPrivate}) + chains = hc.h.policyEngine.APE.(*apeMock).chainMap[engine.ContainerTarget(info.BktInfo.CID.EncodeToString())] + require.Len(t, chains, 2) + require.Len(t, chains[0].Rules, 0) + require.Len(t, chains[1].Rules, 0) + + putBucketACL(hc, bktName, info.Box, map[string]string{api.AmzACL: basicACLPublic}) + chains = hc.h.policyEngine.APE.(*apeMock).chainMap[engine.ContainerTarget(info.BktInfo.CID.EncodeToString())] + require.Len(t, chains, 2) + require.Len(t, chains[0].Rules, 1) + require.Len(t, chains[1].Rules, 1) + + putPublicAccessBlock(hc, bktName, &data.PublicAccessBlockConfiguration{ + IgnorePublicAcls: true, + }, nil) + require.Len(t, hc.h.policyEngine.APE.(*apeMock).chainMap[engine.ContainerTarget(info.BktInfo.CID.EncodeToString())], 0) + + deletePublicAccessBlock(hc, bktName) + chains = hc.h.policyEngine.APE.(*apeMock).chainMap[engine.ContainerTarget(info.BktInfo.CID.EncodeToString())] + require.Len(t, chains, 2) + require.Len(t, chains[0].Rules, 1) + require.Len(t, chains[1].Rules, 1) +} + +func TestCheckRestrictPublicBuckets(t *testing.T) { + hc := prepareHandlerContext(t) + bktName := "bucket-restrict-public-buckets" + + createBucket(hc, bktName) + putPublicAccessBlock(hc, bktName, &data.PublicAccessBlockConfiguration{ + RestrictPublicBuckets: true, + }, nil) + + ctx := context.Background() + ctx = middleware.SetReqInfo(ctx, &middleware.ReqInfo{ + BucketName: bktName, + }) + err := hc.Handler().CheckRestrictPublicBuckets(ctx) + require.NoError(t, err) + + key, err := keys.NewPrivateKey() + require.NoError(t, err) + hc.Handler().frostfsid.(*frostfsidMock).data["devenv"] = key.PublicKey() + putBucketPolicy(hc, bktName, iam.Policy{ // non-public policy + Version: "2012-10-17", + Statement: []iam.Statement{ + { + Principal: map[iam.PrincipalType][]string{iam.AWSPrincipalType: {"arn:aws:iam:::user/devenv"}}, + Effect: "Allow", + Action: []string{"s3:*"}, + Resource: []string{fmt.Sprintf("arn:aws:s3:::%s/*", bktName)}, + }, + }, + }) + err = hc.Handler().CheckRestrictPublicBuckets(ctx) + require.NoError(t, err) + + putBucketPolicy(hc, bktName, iam.Policy{ // public policy + Version: "2012-10-17", + Statement: []iam.Statement{ + { + Principal: map[iam.PrincipalType][]string{iam.Wildcard: {}}, + Effect: "Allow", + Action: []string{"s3:*"}, + Resource: []string{fmt.Sprintf("arn:aws:s3:::%s/*", bktName)}, + }, + }, + }) + err = hc.Handler().CheckRestrictPublicBuckets(ctx) + require.ErrorIs(t, err, apierr.GetAPIError(apierr.ErrAccessDenied)) + + ctx = middleware.SetReqInfo(ctx, &middleware.ReqInfo{ + BucketName: bktName, + UserNamespace: ptr("namespace"), + }) + err = hc.Handler().CheckRestrictPublicBuckets(ctx) + require.ErrorIs(t, err, apierr.GetAPIError(apierr.ErrAccessDenied)) + + ctx = middleware.SetReqInfo(ctx, &middleware.ReqInfo{ + BucketName: bktName, + UserNamespace: ptr(""), + }) + err = hc.Handler().CheckRestrictPublicBuckets(ctx) + require.NoError(t, err) + + deletePublicAccessBlock(hc, bktName) + ctx = middleware.SetReqInfo(ctx, &middleware.ReqInfo{ + BucketName: bktName, + }) + err = hc.Handler().CheckRestrictPublicBuckets(ctx) + require.NoError(t, err) +} + +func putPublicAccessBlock(hc *handlerContext, bktName string, cfg *data.PublicAccessBlockConfiguration, headers map[string]string) { + w := putPublicAccessBlockBase(hc, bktName, cfg, headers) + assertStatus(hc.t, w, http.StatusOK) +} + +func putPublicAccessBlockErr(hc *handlerContext, bktName string, cfg *data.PublicAccessBlockConfiguration, headers map[string]string, err apierr.Error) { + w := putPublicAccessBlockBase(hc, bktName, cfg, headers) + assertS3Error(hc.t, w, err) +} + +func putPublicAccessBlockBase(hc *handlerContext, bktName string, cfg *data.PublicAccessBlockConfiguration, headers map[string]string) *httptest.ResponseRecorder { + w, r := prepareTestRequest(hc, bktName, "", cfg) + for k, v := range headers { + r.Header.Set(k, v) + } + + hc.Handler().PutPublicAccessBlockHandler(w, r) + return w +} + +func getPublicAccessBlock(hc *handlerContext, bktName string) *data.PublicAccessBlockConfiguration { + w := getPublicAccessBlockBase(hc, bktName) + assertStatus(hc.t, w, http.StatusOK) + res := &data.PublicAccessBlockConfiguration{} + parseTestResponse(hc.t, w, res) + return res +} + +func getPublicAccessBlockErr(hc *handlerContext, bktName string, err apierr.Error) { + w := getPublicAccessBlockBase(hc, bktName) + assertS3Error(hc.t, w, err) +} + +func getPublicAccessBlockBase(hc *handlerContext, bktName string) *httptest.ResponseRecorder { + w, r := prepareTestRequest(hc, bktName, "", nil) + hc.Handler().GetPublicAccessBlockHandler(w, r) + return w +} + +func deletePublicAccessBlock(hc *handlerContext, bktName string) { + w := deletePublicAccessBlockBase(hc, bktName) + assertStatus(hc.t, w, http.StatusNoContent) +} + +func deletePublicAccessBlockBase(hc *handlerContext, bktName string) *httptest.ResponseRecorder { + w, r := prepareTestRequest(hc, bktName, "", nil) + hc.Handler().DeletePublicAccessBlockHandler(w, r) + return w +} diff --git a/api/handler/acl.go b/api/handler/acl.go index 1104b2a22..06b9410e2 100644 --- a/api/handler/acl.go +++ b/api/handler/acl.go @@ -177,17 +177,26 @@ func (h *handler) putBucketACLAPEHandler(w http.ResponseWriter, r *http.Request, return } - chainRules := bucketCannedACLToAPERules(cannedACL, reqInfo, bktInfo.CID) - if err = h.policyEngine.APE.SaveACLChains(bktInfo.CID.EncodeToString(), chainRules); err != nil { - h.logAndSendError(ctx, w, "failed to add morph rule chains", reqInfo, err) + if settings.PublicAccessBlock != nil && settings.PublicAccessBlock.BlockPublicAcls && cannedACL != basicACLPrivate { + h.logAndSendError(ctx, w, "public acls are blocked", reqInfo, errors.GetAPIError(errors.ErrAccessDenied)) return } - settings.CannedACL = cannedACL + // Don't set ACL chains if IgnorePublicAcls is set to true and new ACL isn't private + if settings.PublicAccessBlock == nil || !settings.PublicAccessBlock.IgnorePublicAcls || cannedACL == basicACLPrivate { + chainRules := bucketCannedACLToAPERules(cannedACL, reqInfo, bktInfo.CID) + if err = h.policyEngine.APE.SaveACLChains(bktInfo.CID.EncodeToString(), chainRules); err != nil { + h.logAndSendError(ctx, w, "failed to add morph rule chains", reqInfo, err) + return + } + } + + newSettings := *settings + newSettings.CannedACL = cannedACL sp := &layer.PutSettingsParams{ BktInfo: bktInfo, - Settings: settings, + Settings: &newSettings, } if err = h.obj.PutBucketSettings(ctx, sp); err != nil { @@ -258,22 +267,14 @@ func (h *handler) GetBucketPolicyStatusHandler(w http.ResponseWriter, r *http.Re return } - var bktPolicy engineiam.Policy + var bktPolicy s3common.Policy if err = json.Unmarshal(jsonPolicy, &bktPolicy); err != nil { h.logAndSendError(ctx, w, "could not parse bucket policy", reqInfo, err) return } policyStatus := &PolicyStatus{ - IsPublic: PolicyStatusIsPublicFalse, - } - - for _, st := range bktPolicy.Statement { - // https://docs.aws.amazon.com/AmazonS3/latest/userguide/access-control-block-public-access.html#access-control-block-public-access-policy-status - if _, ok := st.Principal[engineiam.Wildcard]; ok { - policyStatus.IsPublic = PolicyStatusIsPublicTrue - break - } + IsPublic: getPolicyStatus(bktPolicy), } if err = middleware.EncodeToResponse(w, policyStatus); err != nil { @@ -282,6 +283,16 @@ func (h *handler) GetBucketPolicyStatusHandler(w http.ResponseWriter, r *http.Re } } +func getPolicyStatus(policy s3common.Policy) PolicyStatusIsPublic { + for _, st := range policy.Statement { + // https://docs.aws.amazon.com/AmazonS3/latest/userguide/access-control-block-public-access.html#access-control-block-public-access-policy-status + if _, ok := st.Principal[s3common.Wildcard]; ok && st.Effect == s3common.AllowEffect { + return PolicyStatusIsPublicTrue + } + } + return PolicyStatusIsPublicFalse +} + func (h *handler) GetBucketPolicyHandler(w http.ResponseWriter, r *http.Request) { ctx, span := tracing.StartSpanFromContext(r.Context(), "handler.GetBucketPolicy") defer span.End() @@ -355,6 +366,12 @@ func (h *handler) PutBucketPolicyHandler(w http.ResponseWriter, r *http.Request) return } + settings, err := h.obj.GetBucketSettings(ctx, bktInfo) + if err != nil { + h.logAndSendError(ctx, w, "couldn't get bucket settings", reqInfo, err) + return + } + jsonPolicy, err := io.ReadAll(r.Body) if err != nil { h.logAndSendError(ctx, w, "read body", reqInfo, err) @@ -367,6 +384,11 @@ func (h *handler) PutBucketPolicyHandler(w http.ResponseWriter, r *http.Request) return } + if settings.PublicAccessBlock != nil && settings.PublicAccessBlock.BlockPublicPolicy && getPolicyStatus(bktPolicy) == PolicyStatusIsPublicTrue { + h.logAndSendError(ctx, w, "public policy is blocked", reqInfo, errors.GetAPIError(errors.ErrAccessDenied)) + return + } + for _, stat := range bktPolicy.Statement { if len(stat.NotResource) != 0 { h.logAndSendError(ctx, w, "policy resource mismatched bucket", reqInfo, errors.GetAPIError(errors.ErrMalformedPolicy)) diff --git a/api/handler/acl_test.go b/api/handler/acl_test.go index eab081afa..c82341f3d 100644 --- a/api/handler/acl_test.go +++ b/api/handler/acl_test.go @@ -358,6 +358,11 @@ func putBucketACL(hc *handlerContext, bktName string, box *accessbox.Box, header assertStatus(hc.t, w, http.StatusOK) } +func putBucketACLErr(hc *handlerContext, bktName string, box *accessbox.Box, header map[string]string, err apierr.Error) { + w := putBucketACLBase(hc, bktName, box, header, nil) + assertS3Error(hc.t, w, err) +} + func putBucketACLAssertS3Error(hc *handlerContext, bktName string, box *accessbox.Box, header map[string]string, body *AccessControlPolicy, code apierr.ErrorCode) { w := putBucketACLBase(hc, bktName, box, header, body) assertS3Error(hc.t, w, apierr.GetAPIError(code)) diff --git a/api/handler/api.go b/api/handler/api.go index f88038644..bb5d0cbeb 100644 --- a/api/handler/api.go +++ b/api/handler/api.go @@ -66,6 +66,7 @@ type ( DeleteBucketPolicy(ns string, cnrID cid.ID, chainIDs []chain.ID) error GetBucketPolicy(ns string, cnrID cid.ID) ([]byte, error) SaveACLChains(cid string, chains []*chain.Chain) error + DeleteACLChains(cid string, chainIDs []chain.ID) error } ) diff --git a/api/handler/handlers_test.go b/api/handler/handlers_test.go index 493cda8ab..e7101bd21 100644 --- a/api/handler/handlers_test.go +++ b/api/handler/handlers_test.go @@ -499,6 +499,20 @@ func (a *apeMock) SaveACLChains(cid string, chains []*chain.Chain) error { return nil } +func (a *apeMock) DeleteACLChains(cid string, chainIDs []chain.ID) error { + if a.err != nil { + return a.err + } + + for i := range chainIDs { + if err := a.RemoveChain(engine.ContainerTarget(cid), chainIDs[i]); err != nil { + return err + } + } + + return nil +} + type frostfsidMock struct { data map[string]*keys.PublicKey } diff --git a/api/handler/locking.go b/api/handler/locking.go index 57b8fe063..3b3a735f2 100644 --- a/api/handler/locking.go +++ b/api/handler/locking.go @@ -100,14 +100,15 @@ func (h *handler) GetBucketObjectLockConfigHandler(w http.ResponseWriter, r *htt return } - if settings.LockConfiguration == nil { - settings.LockConfiguration = &data.ObjectLockConfiguration{} + newSettings := *settings + if newSettings.LockConfiguration == nil { + newSettings.LockConfiguration = &data.ObjectLockConfiguration{} } - if settings.LockConfiguration.ObjectLockEnabled == "" { - settings.LockConfiguration.ObjectLockEnabled = enabledValue + if newSettings.LockConfiguration.ObjectLockEnabled == "" { + newSettings.LockConfiguration.ObjectLockEnabled = enabledValue } - if err = middleware.EncodeToResponse(w, settings.LockConfiguration); err != nil { + if err = middleware.EncodeToResponse(w, newSettings.LockConfiguration); err != nil { h.logAndSendError(ctx, w, "something went wrong", reqInfo, err) } } diff --git a/api/middleware/auth.go b/api/middleware/auth.go index ede724952..e5f6df578 100644 --- a/api/middleware/auth.go +++ b/api/middleware/auth.go @@ -88,7 +88,7 @@ func Auth(center Center, log *zap.Logger) Func { } type FrostFSIDValidator interface { - ValidatePublicKey(key *keys.PublicKey) error + GetUserNamespace(key *keys.PublicKey) (string, error) } func FrostfsIDValidation(frostfsID FrostFSIDValidator, log *zap.Logger) Func { @@ -104,7 +104,8 @@ func FrostfsIDValidation(frostfsID FrostFSIDValidator, log *zap.Logger) Func { return } - if err = validateBearerToken(frostfsID, bd.Gate.BearerToken); err != nil { + namespace, err := getNamespaceFromBearerToken(frostfsID, bd.Gate.BearerToken) + if err != nil { reqLogOrDefault(ctx, log).Error(logs.FrostfsIDValidationFailed, zap.Error(err), logs.TagField(logs.TagDatapath)) if _, wrErr := WriteErrorResponse(w, GetReqInfo(ctx), err); wrErr != nil { reqLogOrDefault(ctx, log).Error(logs.FailedToWriteResponse, zap.Error(wrErr), logs.TagField(logs.TagDatapath)) @@ -112,25 +113,58 @@ func FrostfsIDValidation(frostfsID FrostFSIDValidator, log *zap.Logger) Func { span.End() return } + reqInfo := GetReqInfo(r.Context()) + reqInfo.UserNamespace = &namespace + + span.End() + h.ServeHTTP(w, r) + }) + } +} + +func getNamespaceFromBearerToken(frostfsID FrostFSIDValidator, bt *bearer.Token) (string, error) { + m := new(acl.BearerToken) + bt.WriteToV2(m) + + pk, err := keys.NewPublicKeyFromBytes(m.GetSignature().GetKey(), elliptic.P256()) + if err != nil { + return "", fmt.Errorf("invalid bearer token public key: %w", err) + } + + namespace, err := frostfsID.GetUserNamespace(pk) + if err != nil { + return "", fmt.Errorf("get user namespace: %w", err) + } + + return namespace, nil +} + +type PublicAccessBlockChecker interface { + CheckRestrictPublicBuckets(context.Context) error +} + +func RestrictPublicBuckets(checker PublicAccessBlockChecker, log *zap.Logger) Func { + return func(h http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + ctx, span := tracing.StartSpanFromContext(r.Context(), "middleware.RestrictPublicBuckets") + + if r.Method == http.MethodOptions { + span.End() + h.ServeHTTP(w, r) + return + } + + if err := checker.CheckRestrictPublicBuckets(ctx); err != nil { + reqLogOrDefault(ctx, log).Error(logs.RestrictPublicBucketsCheckFailed, zap.Error(err), logs.TagField(logs.TagDatapath)) + if _, wrErr := WriteErrorResponse(w, GetReqInfo(ctx), err); wrErr != nil { + reqLogOrDefault(ctx, log).Error(logs.FailedToWriteResponse, zap.Error(wrErr), logs.TagField(logs.TagDatapath)) + } + span.End() + return + } span.End() h.ServeHTTP(w, r) }) } } - -func validateBearerToken(frostfsID FrostFSIDValidator, bt *bearer.Token) error { - m := new(acl.BearerToken) - bt.WriteToV2(m) - - pk, err := keys.NewPublicKeyFromBytes(m.GetSignature().GetKey(), elliptic.P256()) - if err != nil { - return fmt.Errorf("invalid bearer token public key: %w", err) - } - - if err = frostfsID.ValidatePublicKey(pk); err != nil { - return fmt.Errorf("validation data user key failed: %w", err) - } - - return nil -} diff --git a/api/middleware/constants.go b/api/middleware/constants.go index 2c8cdfc6f..1aad6c108 100644 --- a/api/middleware/constants.go +++ b/api/middleware/constants.go @@ -48,6 +48,9 @@ const ( DeleteBucketLifecycleOperation = "DeleteBucketLifecycle" DeleteBucketEncryptionOperation = "DeleteBucketEncryption" DeleteBucketOperation = "DeleteBucket" + PutPublicAccessBlockOperation = "PutPublicAccessBlock" + GetPublicAccessBlockOperation = "GetPublicAccessBlock" + DeletePublicAccessBlockOperation = "DeletePublicAccessBlock" // object operations. @@ -78,34 +81,35 @@ const ( ) const ( - UploadsQuery = "uploads" - LocationQuery = "location" - PolicyStatusQuery = "policyStatus" - PolicyQuery = "policy" - LifecycleQuery = "lifecycle" - EncryptionQuery = "encryption" - CorsQuery = "cors" - ACLQuery = "acl" - WebsiteQuery = "website" - AccelerateQuery = "accelerate" - RequestPaymentQuery = "requestPayment" - LoggingQuery = "logging" - ReplicationQuery = "replication" - TaggingQuery = "tagging" - ObjectLockQuery = "object-lock" - VersioningQuery = "versioning" - NotificationQuery = "notification" - EventsQuery = "events" - VersionsQuery = "versions" - ListTypeQuery = "list-type" - MetadataQuery = "metadata" - DeleteQuery = "delete" - UploadIDQuery = "uploadId" - RetentionQuery = "retention" - LegalQuery = "legal" - AttributesQuery = "attributes" - PartNumberQuery = "partNumber" - LegalHoldQuery = "legal-hold" + UploadsQuery = "uploads" + LocationQuery = "location" + PolicyStatusQuery = "policyStatus" + PolicyQuery = "policy" + LifecycleQuery = "lifecycle" + EncryptionQuery = "encryption" + CorsQuery = "cors" + ACLQuery = "acl" + WebsiteQuery = "website" + AccelerateQuery = "accelerate" + RequestPaymentQuery = "requestPayment" + LoggingQuery = "logging" + ReplicationQuery = "replication" + TaggingQuery = "tagging" + ObjectLockQuery = "object-lock" + VersioningQuery = "versioning" + NotificationQuery = "notification" + EventsQuery = "events" + VersionsQuery = "versions" + ListTypeQuery = "list-type" + MetadataQuery = "metadata" + DeleteQuery = "delete" + UploadIDQuery = "uploadId" + RetentionQuery = "retention" + LegalQuery = "legal" + AttributesQuery = "attributes" + PartNumberQuery = "partNumber" + LegalHoldQuery = "legal-hold" + PublicAccessBlockQuery = "publicAccessBlock" ) const ( diff --git a/api/middleware/policy.go b/api/middleware/policy.go index 3193f3843..bb6fba10d 100644 --- a/api/middleware/policy.go +++ b/api/middleware/policy.go @@ -293,6 +293,8 @@ func determineBucketOperation(r *http.Request) string { return true }(): return ListObjectsV1Operation + case query.Has(PublicAccessBlockQuery): + return GetPublicAccessBlockOperation default: return unmatchedBucketOperation } @@ -316,6 +318,8 @@ func determineBucketOperation(r *http.Request) string { return PutBucketVersioningOperation case query.Has(NotificationQuery): return PutBucketNotificationOperation + case query.Has(PublicAccessBlockQuery): + return PutPublicAccessBlockOperation case len(query) == 0: return CreateBucketOperation default: @@ -342,6 +346,8 @@ func determineBucketOperation(r *http.Request) string { return DeleteBucketLifecycleOperation case query.Has(EncryptionQuery): return DeleteBucketEncryptionOperation + case query.Has(PublicAccessBlockQuery): + return DeletePublicAccessBlockOperation case len(query) == 0: return DeleteBucketOperation default: diff --git a/api/middleware/policy_test.go b/api/middleware/policy_test.go index 6a308eb93..c61956f98 100644 --- a/api/middleware/policy_test.go +++ b/api/middleware/policy_test.go @@ -152,6 +152,12 @@ func TestDetermineBucketOperation(t *testing.T) { method: http.MethodGet, expected: ListObjectsV1Operation, }, + { + name: "GetPublicAccessBlockOperation", + method: http.MethodGet, + queryParam: map[string]string{PublicAccessBlockQuery: defaultValue}, + expected: GetPublicAccessBlockOperation, + }, { name: "UnmatchedBucketOperation GET", method: http.MethodGet, @@ -217,6 +223,12 @@ func TestDetermineBucketOperation(t *testing.T) { method: http.MethodPut, expected: CreateBucketOperation, }, + { + name: "PutPublicAccessBlockOperation", + method: http.MethodPut, + queryParam: map[string]string{PublicAccessBlockQuery: defaultValue}, + expected: PutPublicAccessBlockOperation, + }, { name: "UnmatchedBucketOperation PUT", method: http.MethodPut, @@ -275,6 +287,12 @@ func TestDetermineBucketOperation(t *testing.T) { method: http.MethodDelete, expected: DeleteBucketOperation, }, + { + name: "DeletePublicAccessBlockOperation", + method: http.MethodDelete, + queryParam: map[string]string{PublicAccessBlockQuery: defaultValue}, + expected: DeletePublicAccessBlockOperation, + }, { name: "UnmatchedBucketOperation DELETE", method: http.MethodDelete, diff --git a/api/middleware/reqinfo.go b/api/middleware/reqinfo.go index c61563c12..612885120 100644 --- a/api/middleware/reqinfo.go +++ b/api/middleware/reqinfo.go @@ -38,7 +38,8 @@ type ( TraceID string // Trace ID URL *url.URL // Request url Namespace string - User string // User owner id + User string // User owner id + UserNamespace *string // Namespace of user Tagging *data.Tagging RequestVHSEnabled bool RequestType ReqType diff --git a/api/router.go b/api/router.go index b65ff80f1..4ae8debb5 100644 --- a/api/router.go +++ b/api/router.go @@ -88,9 +88,13 @@ type ( ListPartsHandler(w http.ResponseWriter, r *http.Request) ListMultipartUploadsHandler(http.ResponseWriter, *http.Request) PatchObjectHandler(http.ResponseWriter, *http.Request) + PutPublicAccessBlockHandler(http.ResponseWriter, *http.Request) + GetPublicAccessBlockHandler(http.ResponseWriter, *http.Request) + DeletePublicAccessBlockHandler(http.ResponseWriter, *http.Request) ResolveBucket(ctx context.Context, bucket string) (*data.BucketInfo, error) ResolveCID(ctx context.Context, bucket string) (cid.ID, error) + CheckRestrictPublicBuckets(ctx context.Context) error } ) @@ -156,11 +160,11 @@ func NewRouter(cfg Config) *chi.Mux { })) defaultRouter := chi.NewRouter() - defaultRouter.Mount("/{bucket}", bucketRouter(cfg.Handler)) + defaultRouter.Mount("/{bucket}", bucketRouter(cfg.Handler, cfg.Log)) defaultRouter.Get("/", named(s3middleware.ListBucketsOperation, cfg.Handler.ListBucketsHandler)) attachErrorHandler(defaultRouter) - vhsRouter := newDomainRouter(cfg.Handler) + vhsRouter := newDomainRouter(cfg.Handler, cfg.Log) router := newGlobalRouter(defaultRouter, vhsRouter) api.Mount("/", router) @@ -175,7 +179,7 @@ type domainRouter struct { defaultRouter chi.Router } -func newDomainRouter(handler Handler) *domainRouter { +func newDomainRouter(handler Handler, log *zap.Logger) *domainRouter { defaultRouter := chi.NewRouter() defaultRouter.Group(func(r chi.Router) { r.Method(http.MethodGet, "/", NewHandlerFilter(). @@ -188,7 +192,7 @@ func newDomainRouter(handler Handler) *domainRouter { attachErrorHandler(defaultRouter) return &domainRouter{ - bucketRouter: bucketRouter(handler), + bucketRouter: bucketRouter(handler, log), defaultRouter: defaultRouter, } } @@ -289,9 +293,13 @@ func attachErrorHandler(api *chi.Mux) { api.MethodNotAllowed(named("MethodNotAllowed", errorHandler)) } -func bucketRouter(h Handler) chi.Router { +func bucketRouter(h Handler, log *zap.Logger) chi.Router { bktRouter := chi.NewRouter() + bktRouter.Use( + s3middleware.RestrictPublicBuckets(h, log), + ) + bktRouter.Mount("/", objectRouter(h)) bktRouter.Options("/", named(s3middleware.OptionsBucketOperation, h.Preflight)) @@ -368,6 +376,9 @@ func bucketRouter(h Handler) chi.Router { AllowedQueries(s3middleware.QueryDelimiter, s3middleware.QueryMaxKeys, s3middleware.QueryPrefix, s3middleware.QueryMarker, s3middleware.QueryEncodingType). Handler(named(s3middleware.ListObjectsV1Operation, h.ListObjectsV1Handler))). + Add(NewFilter(). + Queries(s3middleware.PublicAccessBlockQuery). + Handler(named(s3middleware.GetPublicAccessBlockOperation, h.GetPublicAccessBlockHandler))). DefaultHandler(notSupportedHandler())) }) @@ -401,6 +412,9 @@ func bucketRouter(h Handler) chi.Router { Add(NewFilter(). Queries(s3middleware.NotificationQuery). Handler(named(s3middleware.PutBucketNotificationOperation, h.PutBucketNotificationHandler))). + Add(NewFilter(). + Queries(s3middleware.PublicAccessBlockQuery). + Handler(named(s3middleware.PutPublicAccessBlockOperation, h.PutPublicAccessBlockHandler))). Add(NewFilter(). NoQueries(). Handler(named(s3middleware.CreateBucketOperation, h.CreateBucketHandler))). @@ -438,6 +452,9 @@ func bucketRouter(h Handler) chi.Router { Add(NewFilter(). Queries(s3middleware.EncryptionQuery). Handler(named(s3middleware.DeleteBucketEncryptionOperation, h.DeleteBucketEncryptionHandler))). + Add(NewFilter(). + Queries(s3middleware.PublicAccessBlockQuery). + Handler(named(s3middleware.DeletePublicAccessBlockOperation, h.DeletePublicAccessBlockHandler))). Add(NewFilter(). NoQueries(). Handler(named(s3middleware.DeleteBucketOperation, h.DeleteBucketHandler))). diff --git a/api/router_mock_test.go b/api/router_mock_test.go index eef15d2e8..c22f93090 100644 --- a/api/router_mock_test.go +++ b/api/router_mock_test.go @@ -133,12 +133,12 @@ type frostFSIDMock struct { userGroupsError bool } -func (f *frostFSIDMock) ValidatePublicKey(*keys.PublicKey) error { +func (f *frostFSIDMock) GetUserNamespace(*keys.PublicKey) (string, error) { if f.validateError { - return fmt.Errorf("some error") + return "", fmt.Errorf("some error") } - return nil + return "", nil } func (f *frostFSIDMock) GetUserGroupIDsAndClaims(util.Uint160) ([]string, map[string]string, error) { @@ -176,9 +176,10 @@ func (m *resourceTaggingMock) GetObjectTagging(context.Context, *data.GetObjectT } type handlerMock struct { - t *testing.T - cfg *middlewareSettingsMock - buckets map[string]*data.BucketInfo + t *testing.T + cfg *middlewareSettingsMock + buckets map[string]*data.BucketInfo + restrict map[string]error } type handlerResult struct { @@ -577,6 +578,37 @@ func (h *handlerMock) PatchObjectHandler(http.ResponseWriter, *http.Request) { panic("implement me") } +func (h *handlerMock) PutPublicAccessBlockHandler(w http.ResponseWriter, r *http.Request) { + cfg := new(data.PublicAccessBlockConfiguration) + err := xml.NewDecoder(r.Body).Decode(cfg) + require.NoError(h.t, err) + + if cfg.RestrictPublicBuckets { + reqInfo := middleware.GetReqInfo(r.Context()) + h.restrict[reqInfo.Namespace+reqInfo.BucketName] = apierr.GetAPIError(apierr.ErrAccessDenied) + } + + res := &handlerResult{ + Method: "PutPublicAccessBlock", + ReqInfo: middleware.GetReqInfo(r.Context()), + } + + h.writeResponse(w, res) +} + +func (h *handlerMock) GetPublicAccessBlockHandler(http.ResponseWriter, *http.Request) { + panic("implement me") +} + +func (h *handlerMock) DeletePublicAccessBlockHandler(http.ResponseWriter, *http.Request) { + panic("implement me") +} + +func (h *handlerMock) CheckRestrictPublicBuckets(ctx context.Context) error { + reqInfo := middleware.GetReqInfo(ctx) + return h.restrict[reqInfo.Namespace+reqInfo.BucketName] +} + func (h *handlerMock) ResolveBucket(ctx context.Context, name string) (*data.BucketInfo, error) { reqInfo := middleware.GetReqInfo(ctx) bktInfo, ok := h.buckets[reqInfo.Namespace+name] diff --git a/api/router_test.go b/api/router_test.go index 14898a530..071ecfd06 100644 --- a/api/router_test.go +++ b/api/router_test.go @@ -67,7 +67,7 @@ func prepareRouter(t *testing.T, opts ...option) *routerMock { Enabled: true, } - handlerTestMock := &handlerMock{t: t, cfg: middlewareSettings, buckets: map[string]*data.BucketInfo{}} + handlerTestMock := &handlerMock{t: t, cfg: middlewareSettings, buckets: map[string]*data.BucketInfo{}, restrict: map[string]error{}} cfg := Config{ Throttle: middleware.ThrottleOpts{ @@ -647,6 +647,41 @@ func TestPreflightWithoutAuth(t *testing.T) { require.Equal(t, http.StatusOK, w.Code) } +func TestRestrictPublicBuckets(t *testing.T) { + router := prepareRouter(t) + router.middlewareSettings.denyByDefault = true + + ns, bktName := "", "bucket" + allowOperations(router, ns, []string{"s3:CreateBucket", "s3:ListBucket", "s3:PutBucketPublicAccessBlock"}, nil) + createBucket(router, ns, bktName) + listObjectsV1(router, ns, bktName, "", "", "") + + putPublicAccessBlock(router, ns, bktName, &data.PublicAccessBlockConfiguration{ + RestrictPublicBuckets: true, + }) + listObjectsV1Err(router, ns, bktName, "", "", "", apierr.ErrAccessDenied) +} + +func putPublicAccessBlock(router *routerMock, namespace, bktName string, cfg *data.PublicAccessBlockConfiguration) handlerResult { + w := putPublicAccessBlockBase(router, namespace, bktName, cfg) + resp := readResponse(router.t, w) + require.Equal(router.t, s3middleware.PutPublicAccessBlockOperation, resp.Method) + return resp +} + +func putPublicAccessBlockBase(router *routerMock, namespace, bktName string, cfg *data.PublicAccessBlockConfiguration) *httptest.ResponseRecorder { + queries := url.Values{} + queries.Add(s3middleware.PublicAccessBlockQuery, "") + + body, err := xml.Marshal(cfg) + require.NoError(router.t, err) + w, r := httptest.NewRecorder(), httptest.NewRequest(http.MethodPut, "/"+bktName, bytes.NewBuffer(body)) + r.URL.RawQuery = queries.Encode() + r.Header.Set(FrostfsNamespaceHeader, namespace) + router.ServeHTTP(w, r) + return w +} + func allowOperations(router *routerMock, ns string, operations []string, conditions engineiam.Conditions) { addPolicy(router, ns, "allow", engineiam.AllowEffect, operations, conditions) } diff --git a/internal/frostfs/frostfsid/frostfsid.go b/internal/frostfs/frostfsid/frostfsid.go index d14f62f09..e46b7238d 100644 --- a/internal/frostfs/frostfsid/frostfsid.go +++ b/internal/frostfs/frostfsid/frostfsid.go @@ -52,9 +52,13 @@ func NewFrostFSID(cfg Config) (*FrostFSID, error) { }, nil } -func (f *FrostFSID) ValidatePublicKey(key *keys.PublicKey) error { - _, err := f.getSubject(key.GetScriptHash()) - return err +func (f *FrostFSID) GetUserNamespace(key *keys.PublicKey) (string, error) { + subj, err := f.getSubject(key.GetScriptHash()) + if err != nil { + return "", err + } + + return subj.Namespace, nil } func (f *FrostFSID) GetUserGroupIDsAndClaims(userHash util.Uint160) ([]string, map[string]string, error) { diff --git a/internal/frostfs/policy/morph_rule_chain_storage.go b/internal/frostfs/policy/morph_rule_chain_storage.go index 4edf0bfc7..a9734fc57 100644 --- a/internal/frostfs/policy/morph_rule_chain_storage.go +++ b/internal/frostfs/policy/morph_rule_chain_storage.go @@ -77,6 +77,7 @@ func (c *MorphRuleChainStorage) ListMorphRuleChains(name chain.Name, target engi func (c *MorphRuleChainStorage) PutBucketPolicy(ns string, cnrID cid.ID, policy []byte, chains []*chain.Chain) error { c.cache.Delete(cache.MorphPolicyCacheKey{Target: engine.ContainerTarget(cnrID.EncodeToString()), Name: chain.S3}) + c.cache.DeleteBucketPolicy(string(getBucketPolicyName(cnrID))) tx := c.contract.StartTx() tx.AddChain(policycontract.IAM, ns, getBucketPolicyName(cnrID), policy) @@ -85,11 +86,21 @@ func (c *MorphRuleChainStorage) PutBucketPolicy(ns string, cnrID cid.ID, policy tx.AddChain(policycontract.Container, cnrID.EncodeToString(), chains[i].ID, chains[i].Bytes()) } - return c.contract.SendTx(tx) + err := c.contract.SendTx(tx) + if err != nil { + return fmt.Errorf("send transaction: %w", err) + } + + if err = c.cache.PutBucketPolicy(string(getBucketPolicyName(cnrID)), policy); err != nil { + c.log.Warn(logs.CouldntCacheListPolicyChains, zap.Error(err), logs.TagField(logs.TagApp)) + } + + return nil } func (c *MorphRuleChainStorage) DeleteBucketPolicy(ns string, cnrID cid.ID, chainIDs []chain.ID) error { c.cache.Delete(cache.MorphPolicyCacheKey{Target: engine.ContainerTarget(cnrID.EncodeToString()), Name: chain.S3}) + c.cache.DeleteBucketPolicy(string(getBucketPolicyName(cnrID))) tx := c.contract.StartTx() for _, chainID := range chainIDs { @@ -101,7 +112,20 @@ func (c *MorphRuleChainStorage) DeleteBucketPolicy(ns string, cnrID cid.ID, chai } func (c *MorphRuleChainStorage) GetBucketPolicy(ns string, cnrID cid.ID) ([]byte, error) { - return c.contract.GetChain(policycontract.IAM, ns, getBucketPolicyName(cnrID)) + if policy := c.cache.GetBucketPolicy(string(getBucketPolicyName(cnrID))); len(policy) > 0 { + return policy, nil + } + + policy, err := c.contract.GetChain(policycontract.IAM, ns, getBucketPolicyName(cnrID)) + if err != nil { + return nil, fmt.Errorf("get chain: %w", err) + } + + if err = c.cache.PutBucketPolicy(string(getBucketPolicyName(cnrID)), policy); err != nil { + c.log.Warn(logs.CouldntCacheListPolicyChains, zap.Error(err), logs.TagField(logs.TagApp)) + } + + return policy, nil } func (c *MorphRuleChainStorage) SaveACLChains(cid string, chains []*chain.Chain) error { @@ -116,6 +140,18 @@ func (c *MorphRuleChainStorage) SaveACLChains(cid string, chains []*chain.Chain) return c.contract.SendTx(tx) } +func (c *MorphRuleChainStorage) DeleteACLChains(cid string, chainIDs []chain.ID) error { + c.cache.Delete(cache.MorphPolicyCacheKey{Target: engine.ContainerTarget(cid), Name: chain.S3}) + c.cache.Delete(cache.MorphPolicyCacheKey{Target: engine.ContainerTarget(cid), Name: chain.Ingress}) + + tx := c.contract.StartTx() + for _, chainID := range chainIDs { + tx.RemoveChain(policycontract.Container, cid, chainID) + } + + return c.contract.SendTx(tx) +} + func getKind(target engine.Target) policycontract.Kind { switch target.Type { case engine.Container: diff --git a/internal/frostfs/policy/storage.go b/internal/frostfs/policy/storage.go index ae43a555e..ba8f07350 100644 --- a/internal/frostfs/policy/storage.go +++ b/internal/frostfs/policy/storage.go @@ -71,3 +71,7 @@ func (s *Storage) GetBucketPolicy(ns string, cnrID cid.ID) ([]byte, error) { func (s *Storage) SaveACLChains(ns string, chains []*chain.Chain) error { return s.morph.SaveACLChains(ns, chains) } + +func (s *Storage) DeleteACLChains(cid string, chainIDs []chain.ID) error { + return s.morph.DeleteACLChains(cid, chainIDs) +} diff --git a/internal/logs/logs.go b/internal/logs/logs.go index 761720ba5..9a9776945 100644 --- a/internal/logs/logs.go +++ b/internal/logs/logs.go @@ -166,6 +166,7 @@ const ( FailedToDiscardPutPayloadProbablyGoroutineLeaks = "failed to discard put payload, probably goroutine leaks" FailedToQueueOldUnversionedObjectToDelete = "failed to queue old unversioned object to delete, removal will be performed in lifecycler" DeleteObjects = "delete objects" + RestrictPublicBucketsCheckFailed = "restrict public buckets check failed" ) // External storage. diff --git a/pkg/service/tree/tree.go b/pkg/service/tree/tree.go index 7b3ae788e..e904c96d4 100644 --- a/pkg/service/tree/tree.go +++ b/pkg/service/tree/tree.go @@ -97,6 +97,7 @@ const ( lockConfigurationKV = "LockConfiguration" oidKV = "OID" cidKV = "CID" + publicAccessBlockKV = "PublicAccessBlock" isCombinedKV = "IsCombined" isUnversionedKV = "IsUnversioned" @@ -536,6 +537,12 @@ func (c *Tree) GetSettingsNode(ctx context.Context, bktInfo *data.BucketInfo) (* } } + if publicAccessBlockValue, ok := node.Get(publicAccessBlockKV); ok { + if settings.PublicAccessBlock, err = parsePublicAccessBlock(publicAccessBlockValue); err != nil { + return nil, fmt.Errorf("settings node: %w", err) + } + } + return settings, nil } @@ -1907,10 +1914,50 @@ func metaFromSettings(settings *data.BucketSettings) map[string]string { if settings.OwnerKey != nil { results[ownerKeyKV] = hex.EncodeToString(settings.OwnerKey.Bytes()) } + if settings.PublicAccessBlock != nil { + results[publicAccessBlockKV] = encodePublicAccessBlock(settings.PublicAccessBlock) + } return results } +func encodePublicAccessBlock(cfg *data.PublicAccessBlockConfiguration) string { + return fmt.Sprintf("%v,%v,%v,%v", cfg.BlockPublicAcls, cfg.BlockPublicPolicy, cfg.IgnorePublicAcls, cfg.RestrictPublicBuckets) +} + +func parsePublicAccessBlock(value string) (*data.PublicAccessBlockConfiguration, error) { + errInvalidFormat := fmt.Errorf("invalid public access block configuration: %s", value) + fields := strings.Split(value, ",") + + if len(fields) != 4 { + return nil, errInvalidFormat + } + + blockPublicAcls, err := strconv.ParseBool(fields[0]) + if err != nil { + return nil, errInvalidFormat + } + blockPublicPolicy, err := strconv.ParseBool(fields[1]) + if err != nil { + return nil, errInvalidFormat + } + ignorePublicAcls, err := strconv.ParseBool(fields[2]) + if err != nil { + return nil, errInvalidFormat + } + restrictPublicBuckets, err := strconv.ParseBool(fields[3]) + if err != nil { + return nil, errInvalidFormat + } + + return &data.PublicAccessBlockConfiguration{ + BlockPublicAcls: blockPublicAcls, + BlockPublicPolicy: blockPublicPolicy, + IgnorePublicAcls: ignorePublicAcls, + RestrictPublicBuckets: restrictPublicBuckets, + }, nil +} + func metaFromMultipart(info *data.MultipartInfo, fileName string) map[string]string { info.Meta[FileNameKey] = fileName info.Meta[uploadIDKV] = info.UploadID