From 4d4114e1d24fb27619c0ff178d45cdfaed3c110d Mon Sep 17 00:00:00 2001 From: Alex Vanin Date: Fri, 10 Feb 2023 15:09:32 +0300 Subject: [PATCH 1/4] [TrueCloudLab#25] Add zone data to BucketInfo Signed-off-by: Alex Vanin --- api/data/info.go | 3 ++- api/layer/container.go | 3 +++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/api/data/info.go b/api/data/info.go index c8f9351..d0580a8 100644 --- a/api/data/info.go +++ b/api/data/info.go @@ -22,7 +22,8 @@ const ( type ( // BucketInfo stores basic bucket data. BucketInfo struct { - Name string + Name string // container name from system attribute + Zone string // container zone from system attribute CID cid.ID Owner user.ID Created time.Time diff --git a/api/layer/container.go b/api/layer/container.go index a9638a5..7dc3e1f 100644 --- a/api/layer/container.go +++ b/api/layer/container.go @@ -5,6 +5,7 @@ import ( "fmt" "strconv" + v2container "github.com/TrueCloudLab/frostfs-api-go/v2/container" "github.com/TrueCloudLab/frostfs-s3-gw/api" "github.com/TrueCloudLab/frostfs-s3-gw/api/data" "github.com/TrueCloudLab/frostfs-s3-gw/api/errors" @@ -56,6 +57,7 @@ func (n *layer) containerInfo(ctx context.Context, idCnr cid.ID) (*data.BucketIn info.Owner = cnr.Owner() if domain := container.ReadDomain(cnr); domain.Name() != "" { info.Name = domain.Name() + info.Zone = domain.Zone() } info.Created = container.CreatedAt(cnr) info.LocationConstraint = cnr.Attribute(attributeLocationConstraint) @@ -114,6 +116,7 @@ func (n *layer) createContainer(ctx context.Context, p *CreateBucketParams) (*da } bktInfo := &data.BucketInfo{ Name: p.Name, + Zone: v2container.SysAttributeZoneDefault, Owner: ownerID, Created: TimeNow(ctx), LocationConstraint: p.LocationConstraint, -- 2.45.2 From 45686c5bdf10423873d4e2fef77c14b243f313f1 Mon Sep 17 00:00:00 2001 From: Alex Vanin Date: Fri, 10 Feb 2023 15:19:29 +0300 Subject: [PATCH 2/4] [TrueCloudLab#25] Process allow and deny lists of zones in bucket head requests Signed-off-by: Alex Vanin --- api/handler/api.go | 2 ++ api/handler/head.go | 29 ++++++++++++++++++++++++++++- api/handler/head_test.go | 20 ++++++++++++++++++++ api/headers.go | 1 + cmd/s3-gw/app.go | 6 ++++++ cmd/s3-gw/app_settings.go | 4 ++++ 6 files changed, 61 insertions(+), 1 deletion(-) diff --git a/api/handler/api.go b/api/handler/api.go index 5253af7..0b64b65 100644 --- a/api/handler/api.go +++ b/api/handler/api.go @@ -29,6 +29,8 @@ type ( DefaultMaxAge int NotificatorEnabled bool CopiesNumber uint32 + ResolveZoneList []string + IsResolveListAllow bool // True if ResolveZoneList contains allowed zones } PlacementPolicy interface { diff --git a/api/handler/head.go b/api/handler/head.go index 0bead4b..58d2d85 100644 --- a/api/handler/head.go +++ b/api/handler/head.go @@ -123,8 +123,13 @@ func (h *handler) HeadBucketHandler(w http.ResponseWriter, r *http.Request) { } w.Header().Set(api.ContainerID, bktInfo.CID.EncodeToString()) - w.Header().Set(api.ContainerName, bktInfo.Name) w.Header().Set(api.AmzBucketRegion, bktInfo.LocationConstraint) + + if isAvailableToResolve(bktInfo.Zone, h.cfg.ResolveZoneList, h.cfg.IsResolveListAllow) { + w.Header().Set(api.ContainerName, bktInfo.Name) + w.Header().Set(api.ContainerZone, bktInfo.Zone) + } + api.WriteResponse(w, http.StatusOK, nil, api.MimeNone) } @@ -158,3 +163,25 @@ func writeLockHeaders(h http.Header, legalHold *data.LegalHold, retention *data. h.Set(api.AmzObjectLockMode, retention.Mode) } } + +func isAvailableToResolve(zone string, list []string, isAllowList bool) bool { + // empty zone means container doesn't have proper system name, + // so we don't have to resolve it + if len(zone) == 0 { + return false + } + + var zoneInList bool + for _, t := range list { + if t == zone { + zoneInList = true + break + } + } + // InList | IsAllowList | Result + // 0 0 1 + // 0 1 0 + // 1 0 0 + // 1 1 1 + return zoneInList == isAllowList +} diff --git a/api/handler/head_test.go b/api/handler/head_test.go index f9d01ad..40078f8 100644 --- a/api/handler/head_test.go +++ b/api/handler/head_test.go @@ -86,6 +86,26 @@ func TestInvalidAccessThroughCache(t *testing.T) { assertStatus(t, w, http.StatusForbidden) } +func TestIsAvailableToResolve(t *testing.T) { + list := []string{"container", "s3"} + + for i, testCase := range [...]struct { + isAllowList bool + list []string + zone string + expected bool + }{ + {isAllowList: true, list: list, zone: "container", expected: true}, + {isAllowList: true, list: list, zone: "sftp", expected: false}, + {isAllowList: false, list: list, zone: "s3", expected: false}, + {isAllowList: false, list: list, zone: "system", expected: true}, + {isAllowList: true, list: list, zone: "", expected: false}, + } { + result := isAvailableToResolve(testCase.zone, testCase.list, testCase.isAllowList) + require.Equal(t, testCase.expected, result, "case %d", i+1) + } +} + func newTestAccessBox(t *testing.T, key *keys.PrivateKey) *accessbox.Box { var err error if key == nil { diff --git a/api/headers.go b/api/headers.go index 8f5307e..db9fc2a 100644 --- a/api/headers.go +++ b/api/headers.go @@ -64,6 +64,7 @@ const ( ContainerID = "X-Container-Id" ContainerName = "X-Container-Name" + ContainerZone = "X-Container-Zone" AccessControlAllowOrigin = "Access-Control-Allow-Origin" AccessControlAllowMethods = "Access-Control-Allow-Methods" diff --git a/cmd/s3-gw/app.go b/cmd/s3-gw/app.go index b1c8c74..5a0b950 100644 --- a/cmd/s3-gw/app.go +++ b/cmd/s3-gw/app.go @@ -642,6 +642,12 @@ func (a *App) initHandler() { cfg.CopiesNumber = val } + cfg.ResolveZoneList = a.cfg.GetStringSlice(cfgResolveBucketAllow) + cfg.IsResolveListAllow = len(cfg.ResolveZoneList) > 0 + if !cfg.IsResolveListAllow { + cfg.ResolveZoneList = a.cfg.GetStringSlice(cfgResolveBucketDeny) + } + var err error a.api, err = handler.New(a.log, a.obj, a.nc, cfg) if err != nil { diff --git a/cmd/s3-gw/app_settings.go b/cmd/s3-gw/app_settings.go index 3cb9fef..cd8a3af 100644 --- a/cmd/s3-gw/app_settings.go +++ b/cmd/s3-gw/app_settings.go @@ -130,6 +130,10 @@ const ( // Settings. // List of allowed AccessKeyID prefixes. cfgAllowedAccessKeyIDPrefixes = "allowed_access_key_id_prefixes" + // Bucket resolving options. + cfgResolveBucketAllow = "resolve_bucket.allow" + cfgResolveBucketDeny = "resolve_bucket.deny" + // envPrefix is an environment variables prefix used for configuration. envPrefix = "S3_GW" ) -- 2.45.2 From 4be66a26b0a29b3732d997d18d67fe2e041a260c Mon Sep 17 00:00:00 2001 From: Alex Vanin Date: Fri, 10 Feb 2023 15:21:25 +0300 Subject: [PATCH 3/4] [TrueCloudLab#25] Update docs and config example Signed-off-by: Alex Vanin --- config/config.env | 4 ++++ config/config.yaml | 5 +++++ docs/configuration.md | 47 +++++++++++++++++++++++++++++-------------- 3 files changed, 41 insertions(+), 15 deletions(-) diff --git a/config/config.env b/config/config.env index 79a0ae0..5f65f84 100644 --- a/config/config.env +++ b/config/config.env @@ -123,3 +123,7 @@ S3_GW_FROSTFS_SET_COPIES_NUMBER=0 # List of allowed AccessKeyID prefixes # If not set, S3 GW will accept all AccessKeyIDs S3_GW_ALLOWED_ACCESS_KEY_ID_PREFIXES=Ck9BHsgKcnwfCTUSFm6pxhoNS4cBqgN2NQ8zVgPjqZDX 3stjWenX15YwYzczMr88gy3CQr4NYFBQ8P7keGzH5QFn + +# List of container NNS zones which are allowed or restricted to resolve with HEAD request +S3_GW_RESOLVE_BUCKET_ALLOW=container +# S3_GW_RESOLVE_BUCKET_DENY= diff --git a/config/config.yaml b/config/config.yaml index 88ed20d..1a535d4 100644 --- a/config/config.yaml +++ b/config/config.yaml @@ -144,3 +144,8 @@ frostfs: allowed_access_key_id_prefixes: - Ck9BHsgKcnwfCTUSFm6pxhoNS4cBqgN2NQ8zVgPjqZDX - 3stjWenX15YwYzczMr88gy3CQr4NYFBQ8P7keGzH5QFn + +resolve_bucket: + allow: + - container + deny: diff --git a/docs/configuration.md b/docs/configuration.md index 5a43885..86f08af 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -168,21 +168,22 @@ There are some custom types used for brevity: ### Structure -| Section | Description | -|--------------------|-------------------------------------------------------------| -| no section | [General parameters](#general-section) | -| `wallet` | [Wallet configuration](#wallet-section) | -| `peers` | [Nodes configuration](#peers-section) | -| `placement_policy` | [Placement policy configuration](#placement_policy-section) | -| `server` | [Server configuration](#server-section) | -| `logger` | [Logger configuration](#logger-section) | -| `tree` | [Tree configuration](#tree-section) | -| `cache` | [Cache configuration](#cache-section) | -| `nats` | [NATS configuration](#nats-section) | -| `cors` | [CORS configuration](#cors-section) | -| `pprof` | [Pprof configuration](#pprof-section) | -| `prometheus` | [Prometheus configuration](#prometheus-section) | -| `frostfs` | [Parameters of requests to FrostFS](#frostfs-section) | +| Section | Description | +|--------------------|----------------------------------------------------------------| +| no section | [General parameters](#general-section) | +| `wallet` | [Wallet configuration](#wallet-section) | +| `peers` | [Nodes configuration](#peers-section) | +| `placement_policy` | [Placement policy configuration](#placement_policy-section) | +| `server` | [Server configuration](#server-section) | +| `logger` | [Logger configuration](#logger-section) | +| `tree` | [Tree configuration](#tree-section) | +| `cache` | [Cache configuration](#cache-section) | +| `nats` | [NATS configuration](#nats-section) | +| `cors` | [CORS configuration](#cors-section) | +| `pprof` | [Pprof configuration](#pprof-section) | +| `prometheus` | [Prometheus configuration](#prometheus-section) | +| `frostfs` | [Parameters of requests to FrostFS](#frostfs-section) | +| `resolve_bucket` | [Bucket name resolving configuration](#resolve_bucket-section) | ### General section @@ -478,3 +479,19 @@ frostfs: | Parameter | Type | Default value | Description | |---------------------|----------|---------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------| | `set_copies_number` | `uint32` | `0` | Number of the object copies to consider PUT to FrostFS successful.
Default value `0` means that object will be processed according to the container's placement policy | + +# `resolve_bucket` section + +Bucket name resolving parameters from and to container ID with `HEAD` request. + +```yaml +resolve_bucket: +allow: + - container +deny: +``` + +| Parameter | Type | Default value | Description | +|-----------|------------|---------------|--------------------------------------------------------------------------------------------------------------------------| +| `allow` | `[]string` | | List of container zones which are available to resolve. Mutual exclusive with `deny` list. Prioritized over `deny` list. | +| `deny` | `[]string` | | List of container zones which are restricted to resolve. Mutual exclusive with `allow` list. | -- 2.45.2 From 6b29ce7e0b4e132593aaf9e3ce22cdbf4febae66 Mon Sep 17 00:00:00 2001 From: Alex Vanin Date: Mon, 13 Feb 2023 18:21:32 +0300 Subject: [PATCH 4/4] [TrueCloudLab#25] Update changelog Signed-off-by: Alex Vanin --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8980de7..484b8ff 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ This document outlines major changes between releases. - Return container name in `head-bucket` response (TrueCloudLab#18) - Billing metrics (TrueCloudLab#5) - Multiple configs support (TrueCloudLab#21) +- Bucket name resolving policy (TrueCloudLab#25) ### Changed - Update neo-go to v0.101.0 (#14) -- 2.45.2