TrueCloudLab/frostfs-s3-gw
15
3
Fork 18
Code Issues 28 Pull requests 4 Projects Releases 17 Packages Activity Actions

Restrict container resolving according to gateway configuration #27

Merged
alexvanin merged 4 commits from feature/25 into master 2023-02-17 10:47:49 +00:00
Conversation 1 Commits 4 Files changed 12 +108 -17
12 changed files with 108 additions and 17 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
CHANGELOG.md
View file

@ -8,6 +8,7 @@ This document outlines major changes between releases.
- Return container name in `head-bucket` response (TrueCloudLab#18)
- Billing metrics (TrueCloudLab#5)
- Multiple configs support (TrueCloudLab#21)
- Bucket name resolving policy (TrueCloudLab#25)
### Changed
- Update neo-go to v0.101.0 (#14)

3
api/data/info.go
View file

@ -22,7 +22,8 @@ const (
type (
// BucketInfo stores basic bucket data.
BucketInfo struct {
Name string
Name string // container name from system attribute
Zone string // container zone from system attribute
CID cid.ID
Owner user.ID
Created time.Time

2
api/handler/api.go
View file

@ -29,6 +29,8 @@ type (
DefaultMaxAge int
NotificatorEnabled bool
CopiesNumber uint32
ResolveZoneList []string
IsResolveListAllow bool // True if ResolveZoneList contains allowed zones
}
PlacementPolicy interface {

29
api/handler/head.go
View file

@ -123,8 +123,13 @@ func (h *handler) HeadBucketHandler(w http.ResponseWriter, r *http.Request) {
}
w.Header().Set(api.ContainerID, bktInfo.CID.EncodeToString())
w.Header().Set(api.ContainerName, bktInfo.Name)
w.Header().Set(api.AmzBucketRegion, bktInfo.LocationConstraint)
if isAvailableToResolve(bktInfo.Zone, h.cfg.ResolveZoneList, h.cfg.IsResolveListAllow) {
w.Header().Set(api.ContainerName, bktInfo.Name)
w.Header().Set(api.ContainerZone, bktInfo.Zone)
}
api.WriteResponse(w, http.StatusOK, nil, api.MimeNone)
}
@ -158,3 +163,25 @@ func writeLockHeaders(h http.Header, legalHold *data.LegalHold, retention *data.
h.Set(api.AmzObjectLockMode, retention.Mode)
}
}
func isAvailableToResolve(zone string, list []string, isAllowList bool) bool {
// empty zone means container doesn't have proper system name,
// so we don't have to resolve it
if len(zone) == 0 {
return false
}
var zoneInList bool
for _, t := range list {
if t == zone {
zoneInList = true
break
}
}
// InList | IsAllowList | Result
// 0 0 1
// 0 1 0
// 1 0 0
// 1 1 1
return zoneInList == isAllowList
}

20
api/handler/head_test.go
View file

@ -86,6 +86,26 @@ func TestInvalidAccessThroughCache(t *testing.T) {
assertStatus(t, w, http.StatusForbidden)
}
func TestIsAvailableToResolve(t *testing.T) {
list := []string{"container", "s3"}
for i, testCase := range [...]struct {
isAllowList bool
list []string
zone string
expected bool
}{
{isAllowList: true, list: list, zone: "container", expected: true},
{isAllowList: true, list: list, zone: "sftp", expected: false},
{isAllowList: false, list: list, zone: "s3", expected: false},
{isAllowList: false, list: list, zone: "system", expected: true},
{isAllowList: true, list: list, zone: "", expected: false},
} {
result := isAvailableToResolve(testCase.zone, testCase.list, testCase.isAllowList)
require.Equal(t, testCase.expected, result, "case %d", i+1)
}
}
func newTestAccessBox(t *testing.T, key *keys.PrivateKey) *accessbox.Box {
var err error
if key == nil {

1
api/headers.go
View file

@ -64,6 +64,7 @@ const (
ContainerID = "X-Container-Id"
ContainerName = "X-Container-Name"
ContainerZone = "X-Container-Zone"
AccessControlAllowOrigin = "Access-Control-Allow-Origin"
AccessControlAllowMethods = "Access-Control-Allow-Methods"

3
api/layer/container.go
View file

@ -5,6 +5,7 @@ import (
"fmt"
"strconv"
v2container "github.com/TrueCloudLab/frostfs-api-go/v2/container"
"github.com/TrueCloudLab/frostfs-s3-gw/api"
"github.com/TrueCloudLab/frostfs-s3-gw/api/data"
"github.com/TrueCloudLab/frostfs-s3-gw/api/errors"
@ -56,6 +57,7 @@ func (n *layer) containerInfo(ctx context.Context, idCnr cid.ID) (*data.BucketIn
info.Owner = cnr.Owner()
if domain := container.ReadDomain(cnr); domain.Name() != "" {
info.Name = domain.Name()
info.Zone = domain.Zone()
}
info.Created = container.CreatedAt(cnr)
info.LocationConstraint = cnr.Attribute(attributeLocationConstraint)
@ -114,6 +116,7 @@ func (n *layer) createContainer(ctx context.Context, p *CreateBucketParams) (*da
}
bktInfo := &data.BucketInfo{
Name: p.Name,
Zone: v2container.SysAttributeZoneDefault,
Owner: ownerID,
Created: TimeNow(ctx),
LocationConstraint: p.LocationConstraint,

6
cmd/s3-gw/app.go
View file

@ -642,6 +642,12 @@ func (a *App) initHandler() {
cfg.CopiesNumber = val
}
cfg.ResolveZoneList = a.cfg.GetStringSlice(cfgResolveBucketAllow)
cfg.IsResolveListAllow = len(cfg.ResolveZoneList) > 0
if !cfg.IsResolveListAllow {
cfg.ResolveZoneList = a.cfg.GetStringSlice(cfgResolveBucketDeny)
}
var err error
a.api, err = handler.New(a.log, a.obj, a.nc, cfg)
if err != nil {

4
cmd/s3-gw/app_settings.go
View file

@ -130,6 +130,10 @@ const ( // Settings.
// List of allowed AccessKeyID prefixes.
cfgAllowedAccessKeyIDPrefixes = "allowed_access_key_id_prefixes"
// Bucket resolving options.
cfgResolveBucketAllow = "resolve_bucket.allow"
cfgResolveBucketDeny = "resolve_bucket.deny"
// envPrefix is an environment variables prefix used for configuration.
envPrefix = "S3_GW"
)

4
config/config.env
View file

@ -123,3 +123,7 @@ S3_GW_FROSTFS_SET_COPIES_NUMBER=0
# List of allowed AccessKeyID prefixes
# If not set, S3 GW will accept all AccessKeyIDs
S3_GW_ALLOWED_ACCESS_KEY_ID_PREFIXES=Ck9BHsgKcnwfCTUSFm6pxhoNS4cBqgN2NQ8zVgPjqZDX 3stjWenX15YwYzczMr88gy3CQr4NYFBQ8P7keGzH5QFn
# List of container NNS zones which are allowed or restricted to resolve with HEAD request
S3_GW_RESOLVE_BUCKET_ALLOW=container
# S3_GW_RESOLVE_BUCKET_DENY=

5
config/config.yaml
View file

@ -144,3 +144,8 @@ frostfs:
allowed_access_key_id_prefixes:
- Ck9BHsgKcnwfCTUSFm6pxhoNS4cBqgN2NQ8zVgPjqZDX
- 3stjWenX15YwYzczMr88gy3CQr4NYFBQ8P7keGzH5QFn
resolve_bucket:
KirillovDenis commented 2023-02-10 14:40:04 +00:00 (Migrated from github.com)
Review
Copy link

It seems we should add the same example for config/config.env file

It seems we should add the same example for `config/config.env` file
allow:
- container
deny:

47
docs/configuration.md
View file

@ -168,21 +168,22 @@ There are some custom types used for brevity:
### Structure
| Section | Description |
|--------------------|-------------------------------------------------------------|
| no section | [General parameters](#general-section) |
| `wallet` | [Wallet configuration](#wallet-section) |
| `peers` | [Nodes configuration](#peers-section) |
| `placement_policy` | [Placement policy configuration](#placement_policy-section) |
| `server` | [Server configuration](#server-section) |
| `logger` | [Logger configuration](#logger-section) |
| `tree` | [Tree configuration](#tree-section) |
| `cache` | [Cache configuration](#cache-section) |
| `nats` | [NATS configuration](#nats-section) |
| `cors` | [CORS configuration](#cors-section) |
| `pprof` | [Pprof configuration](#pprof-section) |
| `prometheus` | [Prometheus configuration](#prometheus-section) |
| `frostfs` | [Parameters of requests to FrostFS](#frostfs-section) |
| Section | Description |
|--------------------|----------------------------------------------------------------|
| no section | [General parameters](#general-section) |
| `wallet` | [Wallet configuration](#wallet-section) |
| `peers` | [Nodes configuration](#peers-section) |
| `placement_policy` | [Placement policy configuration](#placement_policy-section) |
| `server` | [Server configuration](#server-section) |
| `logger` | [Logger configuration](#logger-section) |
| `tree` | [Tree configuration](#tree-section) |
| `cache` | [Cache configuration](#cache-section) |
| `nats` | [NATS configuration](#nats-section) |
| `cors` | [CORS configuration](#cors-section) |
| `pprof` | [Pprof configuration](#pprof-section) |
| `prometheus` | [Prometheus configuration](#prometheus-section) |
| `frostfs` | [Parameters of requests to FrostFS](#frostfs-section) |
| `resolve_bucket` | [Bucket name resolving configuration](#resolve_bucket-section) |
### General section
@ -478,3 +479,19 @@ frostfs:
| Parameter | Type | Default value | Description |
|---------------------|----------|---------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
| `set_copies_number` | `uint32` | `0` | Number of the object copies to consider PUT to FrostFS successful. <br/>Default value `0` means that object will be processed according to the container's placement policy |
# `resolve_bucket` section
Bucket name resolving parameters from and to container ID with `HEAD` request.
```yaml
resolve_bucket:
allow:
- container
deny:
```
| Parameter | Type | Default value | Description |
|-----------|------------|---------------|--------------------------------------------------------------------------------------------------------------------------|
| `allow` | `[]string` | | List of container zones which are available to resolve. Mutual exclusive with `deny` list. Prioritized over `deny` list. |
| `deny` | `[]string` | | List of container zones which are restricted to resolve. Mutual exclusive with `allow` list. |