Restrict container resolving according to gateway configuration #27

Merged
alexvanin merged 4 commits from feature/25 into master 2023-02-17 10:47:49 +00:00
12 changed files with 108 additions and 17 deletions

View file

@ -8,6 +8,7 @@ This document outlines major changes between releases.
- Return container name in `head-bucket` response (TrueCloudLab#18) - Return container name in `head-bucket` response (TrueCloudLab#18)
- Billing metrics (TrueCloudLab#5) - Billing metrics (TrueCloudLab#5)
- Multiple configs support (TrueCloudLab#21) - Multiple configs support (TrueCloudLab#21)
- Bucket name resolving policy (TrueCloudLab#25)
### Changed ### Changed
- Update neo-go to v0.101.0 (#14) - Update neo-go to v0.101.0 (#14)

View file

@ -22,7 +22,8 @@ const (
type ( type (
// BucketInfo stores basic bucket data. // BucketInfo stores basic bucket data.
BucketInfo struct { BucketInfo struct {
Name string Name string // container name from system attribute
Zone string // container zone from system attribute
CID cid.ID CID cid.ID
Owner user.ID Owner user.ID
Created time.Time Created time.Time

View file

@ -29,6 +29,8 @@ type (
DefaultMaxAge int DefaultMaxAge int
NotificatorEnabled bool NotificatorEnabled bool
CopiesNumber uint32 CopiesNumber uint32
ResolveZoneList []string
IsResolveListAllow bool // True if ResolveZoneList contains allowed zones
} }
PlacementPolicy interface { PlacementPolicy interface {

View file

@ -123,8 +123,13 @@ func (h *handler) HeadBucketHandler(w http.ResponseWriter, r *http.Request) {
} }
w.Header().Set(api.ContainerID, bktInfo.CID.EncodeToString()) w.Header().Set(api.ContainerID, bktInfo.CID.EncodeToString())
w.Header().Set(api.ContainerName, bktInfo.Name)
w.Header().Set(api.AmzBucketRegion, bktInfo.LocationConstraint) w.Header().Set(api.AmzBucketRegion, bktInfo.LocationConstraint)
if isAvailableToResolve(bktInfo.Zone, h.cfg.ResolveZoneList, h.cfg.IsResolveListAllow) {
w.Header().Set(api.ContainerName, bktInfo.Name)
w.Header().Set(api.ContainerZone, bktInfo.Zone)
}
api.WriteResponse(w, http.StatusOK, nil, api.MimeNone) api.WriteResponse(w, http.StatusOK, nil, api.MimeNone)
} }
@ -158,3 +163,25 @@ func writeLockHeaders(h http.Header, legalHold *data.LegalHold, retention *data.
h.Set(api.AmzObjectLockMode, retention.Mode) h.Set(api.AmzObjectLockMode, retention.Mode)
} }
} }
func isAvailableToResolve(zone string, list []string, isAllowList bool) bool {
// empty zone means container doesn't have proper system name,
// so we don't have to resolve it
if len(zone) == 0 {
return false
}
var zoneInList bool
for _, t := range list {
if t == zone {
zoneInList = true
break
}
}
// InList | IsAllowList | Result
// 0 0 1
// 0 1 0
// 1 0 0
// 1 1 1
return zoneInList == isAllowList
}

View file

@ -86,6 +86,26 @@ func TestInvalidAccessThroughCache(t *testing.T) {
assertStatus(t, w, http.StatusForbidden) assertStatus(t, w, http.StatusForbidden)
} }
func TestIsAvailableToResolve(t *testing.T) {
list := []string{"container", "s3"}
for i, testCase := range [...]struct {
isAllowList bool
list []string
zone string
expected bool
}{
{isAllowList: true, list: list, zone: "container", expected: true},
{isAllowList: true, list: list, zone: "sftp", expected: false},
{isAllowList: false, list: list, zone: "s3", expected: false},
{isAllowList: false, list: list, zone: "system", expected: true},
{isAllowList: true, list: list, zone: "", expected: false},
} {
result := isAvailableToResolve(testCase.zone, testCase.list, testCase.isAllowList)
require.Equal(t, testCase.expected, result, "case %d", i+1)
}
}
func newTestAccessBox(t *testing.T, key *keys.PrivateKey) *accessbox.Box { func newTestAccessBox(t *testing.T, key *keys.PrivateKey) *accessbox.Box {
var err error var err error
if key == nil { if key == nil {

View file

@ -64,6 +64,7 @@ const (
ContainerID = "X-Container-Id" ContainerID = "X-Container-Id"
ContainerName = "X-Container-Name" ContainerName = "X-Container-Name"
ContainerZone = "X-Container-Zone"
AccessControlAllowOrigin = "Access-Control-Allow-Origin" AccessControlAllowOrigin = "Access-Control-Allow-Origin"
AccessControlAllowMethods = "Access-Control-Allow-Methods" AccessControlAllowMethods = "Access-Control-Allow-Methods"

View file

@ -5,6 +5,7 @@ import (
"fmt" "fmt"
"strconv" "strconv"
v2container "github.com/TrueCloudLab/frostfs-api-go/v2/container"
"github.com/TrueCloudLab/frostfs-s3-gw/api" "github.com/TrueCloudLab/frostfs-s3-gw/api"
"github.com/TrueCloudLab/frostfs-s3-gw/api/data" "github.com/TrueCloudLab/frostfs-s3-gw/api/data"
"github.com/TrueCloudLab/frostfs-s3-gw/api/errors" "github.com/TrueCloudLab/frostfs-s3-gw/api/errors"
@ -56,6 +57,7 @@ func (n *layer) containerInfo(ctx context.Context, idCnr cid.ID) (*data.BucketIn
info.Owner = cnr.Owner() info.Owner = cnr.Owner()
if domain := container.ReadDomain(cnr); domain.Name() != "" { if domain := container.ReadDomain(cnr); domain.Name() != "" {
info.Name = domain.Name() info.Name = domain.Name()
info.Zone = domain.Zone()
} }
info.Created = container.CreatedAt(cnr) info.Created = container.CreatedAt(cnr)
info.LocationConstraint = cnr.Attribute(attributeLocationConstraint) info.LocationConstraint = cnr.Attribute(attributeLocationConstraint)
@ -114,6 +116,7 @@ func (n *layer) createContainer(ctx context.Context, p *CreateBucketParams) (*da
} }
bktInfo := &data.BucketInfo{ bktInfo := &data.BucketInfo{
Name: p.Name, Name: p.Name,
Zone: v2container.SysAttributeZoneDefault,
Owner: ownerID, Owner: ownerID,
Created: TimeNow(ctx), Created: TimeNow(ctx),
LocationConstraint: p.LocationConstraint, LocationConstraint: p.LocationConstraint,

View file

@ -642,6 +642,12 @@ func (a *App) initHandler() {
cfg.CopiesNumber = val cfg.CopiesNumber = val
} }
cfg.ResolveZoneList = a.cfg.GetStringSlice(cfgResolveBucketAllow)
cfg.IsResolveListAllow = len(cfg.ResolveZoneList) > 0
if !cfg.IsResolveListAllow {
cfg.ResolveZoneList = a.cfg.GetStringSlice(cfgResolveBucketDeny)
}
var err error var err error
a.api, err = handler.New(a.log, a.obj, a.nc, cfg) a.api, err = handler.New(a.log, a.obj, a.nc, cfg)
if err != nil { if err != nil {

View file

@ -130,6 +130,10 @@ const ( // Settings.
// List of allowed AccessKeyID prefixes. // List of allowed AccessKeyID prefixes.
cfgAllowedAccessKeyIDPrefixes = "allowed_access_key_id_prefixes" cfgAllowedAccessKeyIDPrefixes = "allowed_access_key_id_prefixes"
// Bucket resolving options.
cfgResolveBucketAllow = "resolve_bucket.allow"
cfgResolveBucketDeny = "resolve_bucket.deny"
// envPrefix is an environment variables prefix used for configuration. // envPrefix is an environment variables prefix used for configuration.
envPrefix = "S3_GW" envPrefix = "S3_GW"
) )

View file

@ -123,3 +123,7 @@ S3_GW_FROSTFS_SET_COPIES_NUMBER=0
# List of allowed AccessKeyID prefixes # List of allowed AccessKeyID prefixes
# If not set, S3 GW will accept all AccessKeyIDs # If not set, S3 GW will accept all AccessKeyIDs
S3_GW_ALLOWED_ACCESS_KEY_ID_PREFIXES=Ck9BHsgKcnwfCTUSFm6pxhoNS4cBqgN2NQ8zVgPjqZDX 3stjWenX15YwYzczMr88gy3CQr4NYFBQ8P7keGzH5QFn S3_GW_ALLOWED_ACCESS_KEY_ID_PREFIXES=Ck9BHsgKcnwfCTUSFm6pxhoNS4cBqgN2NQ8zVgPjqZDX 3stjWenX15YwYzczMr88gy3CQr4NYFBQ8P7keGzH5QFn
# List of container NNS zones which are allowed or restricted to resolve with HEAD request
S3_GW_RESOLVE_BUCKET_ALLOW=container
# S3_GW_RESOLVE_BUCKET_DENY=

View file

@ -144,3 +144,8 @@ frostfs:
allowed_access_key_id_prefixes: allowed_access_key_id_prefixes:
- Ck9BHsgKcnwfCTUSFm6pxhoNS4cBqgN2NQ8zVgPjqZDX - Ck9BHsgKcnwfCTUSFm6pxhoNS4cBqgN2NQ8zVgPjqZDX
- 3stjWenX15YwYzczMr88gy3CQr4NYFBQ8P7keGzH5QFn - 3stjWenX15YwYzczMr88gy3CQr4NYFBQ8P7keGzH5QFn
resolve_bucket:
KirillovDenis commented 2023-02-10 14:40:04 +00:00 (Migrated from github.com)
Review

It seems we should add the same example for config/config.env file

It seems we should add the same example for `config/config.env` file
allow:
- container
deny:

View file

@ -169,7 +169,7 @@ There are some custom types used for brevity:
### Structure ### Structure
| Section | Description | | Section | Description |
|--------------------|-------------------------------------------------------------| |--------------------|----------------------------------------------------------------|
| no section | [General parameters](#general-section) | | no section | [General parameters](#general-section) |
| `wallet` | [Wallet configuration](#wallet-section) | | `wallet` | [Wallet configuration](#wallet-section) |
| `peers` | [Nodes configuration](#peers-section) | | `peers` | [Nodes configuration](#peers-section) |
@ -183,6 +183,7 @@ There are some custom types used for brevity:
| `pprof` | [Pprof configuration](#pprof-section) | | `pprof` | [Pprof configuration](#pprof-section) |
| `prometheus` | [Prometheus configuration](#prometheus-section) | | `prometheus` | [Prometheus configuration](#prometheus-section) |
| `frostfs` | [Parameters of requests to FrostFS](#frostfs-section) | | `frostfs` | [Parameters of requests to FrostFS](#frostfs-section) |
| `resolve_bucket` | [Bucket name resolving configuration](#resolve_bucket-section) |
### General section ### General section
@ -478,3 +479,19 @@ frostfs:
| Parameter | Type | Default value | Description | | Parameter | Type | Default value | Description |
|---------------------|----------|---------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------| |---------------------|----------|---------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
| `set_copies_number` | `uint32` | `0` | Number of the object copies to consider PUT to FrostFS successful. <br/>Default value `0` means that object will be processed according to the container's placement policy | | `set_copies_number` | `uint32` | `0` | Number of the object copies to consider PUT to FrostFS successful. <br/>Default value `0` means that object will be processed according to the container's placement policy |
# `resolve_bucket` section
Bucket name resolving parameters from and to container ID with `HEAD` request.
```yaml
resolve_bucket:
allow:
- container
deny:
```
| Parameter | Type | Default value | Description |
|-----------|------------|---------------|--------------------------------------------------------------------------------------------------------------------------|
| `allow` | `[]string` | | List of container zones which are available to resolve. Mutual exclusive with `deny` list. Prioritized over `deny` list. |
| `deny` | `[]string` | | List of container zones which are restricted to resolve. Mutual exclusive with `allow` list. |