TrueCloudLab/frostfs-s3-gw
18
3
Fork 20
Code Issues 29 Pull requests 8 Projects Releases 23 Packages Activity Actions

[#260] Use namespace as domain when resolve bucket

Browse source 
Signed-off-by: Denis Kirillov <d.kirillov@yadro.com>
This commit is contained in:
Denis Kirillov 2023-11-16 15:10:51 +03:00
parent a61ff3b8cb
commit 055cc6a22a
16 changed files with 159 additions and 46 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

11
api/cache/buckets.go vendored
View file

@ -39,7 +39,8 @@ func NewBucketCache(config *Config) *BucketCache {
}
// Get returns a cached object.
func (o *BucketCache) Get(key string) *data.BucketInfo {
func (o *BucketCache) Get(ns, bktName string) *data.BucketInfo {
key := ns + "/" + bktName
entry, err := o.cache.Get(key)
if err != nil {
return nil
@ -56,11 +57,11 @@ func (o *BucketCache) Get(key string) *data.BucketInfo {
}
// Put puts an object to cache.
func (o *BucketCache) Put(bkt *data.BucketInfo) error {
return o.cache.Set(bkt.Name, bkt)
func (o *BucketCache) Put(ns string, bkt *data.BucketInfo) error {
return o.cache.Set(ns+"/"+bkt.Name, bkt)
}
// Delete deletes an object from cache.
func (o *BucketCache) Delete(key string) bool {
return o.cache.Remove(key)
func (o *BucketCache) Delete(ns, bktName string) bool {
return o.cache.Remove(ns + "/" + bktName)
}

8
api/cache/cache_test.go vendored
View file

@ -36,15 +36,15 @@ func TestBucketsCacheType(t *testing.T) {
bktInfo := &data.BucketInfo{Name: "bucket"}
err := cache.Put(bktInfo)
err := cache.Put("", bktInfo)
require.NoError(t, err)
val := cache.Get(bktInfo.Name)
val := cache.Get("", bktInfo.Name)
require.Equal(t, bktInfo, val)
require.Equal(t, 0, observedLog.Len())
err = cache.cache.Set(bktInfo.Name, "tmp")
err = cache.cache.Set("/"+bktInfo.Name, "tmp")
require.NoError(t, err)
assertInvalidCacheEntry(t, cache.Get(bktInfo.Name), observedLog)
assertInvalidCacheEntry(t, cache.Get("", bktInfo.Name), observedLog)
}
func TestObjectNamesCacheType(t *testing.T) {

13
api/layer/cache.go
View file

@ -56,21 +56,22 @@ func NewCache(cfg *CachesConfig) *Cache {
}
}
func (c *Cache) GetBucket(name string) *data.BucketInfo {
return c.bucketCache.Get(name)
func (c *Cache) GetBucket(ns, name string) *data.BucketInfo {
return c.bucketCache.Get(ns, name)
}
func (c *Cache) PutBucket(bktInfo *data.BucketInfo) {
if err := c.bucketCache.Put(bktInfo); err != nil {
func (c *Cache) PutBucket(ns string, bktInfo *data.BucketInfo) {
if err := c.bucketCache.Put(ns, bktInfo); err != nil {
c.logger.Warn(logs.CouldntPutBucketInfoIntoCache,
zap.String("namespace", ns),
zap.String("bucket name", bktInfo.Name),
zap.Stringer("bucket cid", bktInfo.CID),
zap.Error(err))
}
}
func (c *Cache) DeleteBucket(name string) {
c.bucketCache.Delete(name)
func (c *Cache) DeleteBucket(ns, name string) {
c.bucketCache.Delete(ns, name)
}
func (c *Cache) CleanListCacheEntriesContainingObject(objectName string, cnrID cid.ID) {

7
api/layer/container.go
View file

@ -9,6 +9,7 @@ import (
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
s3errors "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/middleware"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container"
@ -41,6 +42,8 @@ func (n *layer) containerInfo(ctx context.Context, idCnr cid.ID) (*data.BucketIn
CID: idCnr,
Name: idCnr.EncodeToString(),
}
reqInfo = middleware.GetReqInfo(ctx)
)
res, err = n.frostFS.Container(ctx, idCnr)
if err != nil {
@ -72,7 +75,7 @@ func (n *layer) containerInfo(ctx context.Context, idCnr cid.ID) (*data.BucketIn
}
}
n.cache.PutBucket(info)
n.cache.PutBucket(reqInfo.Namespace, info)
return info, nil
}
@ -142,7 +145,7 @@ func (n *layer) createContainer(ctx context.Context, p *CreateBucketParams) (*da
return nil, fmt.Errorf("set container eacl: %w", err)
}
n.cache.PutBucket(bktInfo)
n.cache.PutBucket(bktInfo.Zone, bktInfo)
return bktInfo, nil
}

8
api/layer/layer.go
View file

@ -401,7 +401,9 @@ func (n *layer) GetBucketInfo(ctx context.Context, name string) (*data.BucketInf
return nil, fmt.Errorf("unescape bucket name: %w", err)
}
if bktInfo := n.cache.GetBucket(name); bktInfo != nil {
reqInfo := middleware.GetReqInfo(ctx)
if bktInfo := n.cache.GetBucket(reqInfo.Namespace, name); bktInfo != nil {
return bktInfo, nil
}
@ -814,6 +816,8 @@ func (n *layer) DeleteBucket(ctx context.Context, p *DeleteBucketParams) error {
return errors.GetAPIError(errors.ErrBucketNotEmpty)
}
n.cache.DeleteBucket(p.BktInfo.Name)
reqInfo := middleware.GetReqInfo(ctx)
n.cache.DeleteBucket(reqInfo.Namespace, p.BktInfo.Name)
return n.frostFS.DeleteContainer(ctx, p.BktInfo.CID, p.SessionToken)
}

8
api/middleware/reqinfo.go
View file

@ -37,6 +37,7 @@ type (
ObjectName string // Object name
TraceID string // Trace ID
URL *url.URL // Request url
Namespace string
tags []KeyVal // Any additional info not accommodated by above fields
}
@ -186,7 +187,11 @@ func GetReqLog(ctx context.Context) *zap.Logger {
return nil
}
func Request(log *zap.Logger) Func {
type RequestSettings interface {
NamespaceHeader() string
}
func Request(log *zap.Logger, settings RequestSettings) Func {
return func(h http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
// generate random UUIDv4
@ -200,6 +205,7 @@ func Request(log *zap.Logger) Func {
// set request info into context
// bucket name and object will be set in reqInfo later (limitation of go-chi)
reqInfo := NewReqInfo(w, r, ObjectRequest{})
reqInfo.Namespace = r.Header.Get(settings.NamespaceHeader())
r = r.WithContext(SetReqInfo(r.Context(), reqInfo))
// set request id into gRPC meta header

46
api/resolver/resolver.go
View file

@ -6,9 +6,11 @@ import (
"fmt"
"sync"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/middleware"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container"
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/ns"
"golang.org/x/exp/slices"
)
const (
@ -16,6 +18,8 @@ const (
DNSResolver = "dns"
)
const nsDomain = ".ns"
// ErrNoResolvers returns when trying to resolve container without any resolver.
var ErrNoResolvers = errors.New("no resolvers")
@ -28,14 +32,20 @@ type FrostFS interface {
SystemDNS(context.Context) (string, error)
}
type Settings interface {
DefaultNamespaces() []string
}
type Config struct {
FrostFS FrostFS
RPCAddress string
Settings Settings
}
type BucketResolver struct {
rpcAddress string
frostfs FrostFS
settings Settings
mu sync.RWMutex
resolvers []*Resolver
@ -113,7 +123,13 @@ func (r *BucketResolver) UpdateResolvers(resolverNames []string) error {
return nil
}
resolvers, err := createResolvers(resolverNames, &Config{FrostFS: r.frostfs, RPCAddress: r.rpcAddress})
cfg := &Config{
FrostFS: r.frostfs,
RPCAddress: r.rpcAddress,
Settings: r.settings,
}
resolvers, err := createResolvers(resolverNames, cfg)
if err != nil {
return err
}
@ -139,26 +155,34 @@ func (r *BucketResolver) equals(resolverNames []string) bool {
func newResolver(name string, cfg *Config) (*Resolver, error) {
switch name {
case DNSResolver:
return NewDNSResolver(cfg.FrostFS)
return NewDNSResolver(cfg.FrostFS, cfg.Settings)
case NNSResolver:
return NewNNSResolver(cfg.RPCAddress)
return NewNNSResolver(cfg.RPCAddress, cfg.Settings)
default:
return nil, fmt.Errorf("unknown resolver: %s", name)
}
}
func NewDNSResolver(frostFS FrostFS) (*Resolver, error) {
func NewDNSResolver(frostFS FrostFS, settings Settings) (*Resolver, error) {
if frostFS == nil {
return nil, fmt.Errorf("pool must not be nil for DNS resolver")
}
if settings == nil {
return nil, fmt.Errorf("resolver settings must not be nil for DNS resolver")
}
var dns ns.DNS
resolveFunc := func(ctx context.Context, name string) (cid.ID, error) {
domain, err := frostFS.SystemDNS(ctx)
var err error
reqInfo := middleware.GetReqInfo(ctx)
domain := reqInfo.Namespace + nsDomain
if slices.Contains(settings.DefaultNamespaces(), domain) {
domain, err = frostFS.SystemDNS(ctx)
if err != nil {
return cid.ID{}, fmt.Errorf("read system DNS parameter of the FrostFS: %w", err)
}
}
domain = name + "." + domain
cnrID, err := dns.ResolveContainerName(domain)
@ -174,10 +198,13 @@ func NewDNSResolver(frostFS FrostFS) (*Resolver, error) {
}, nil
}
func NewNNSResolver(address string) (*Resolver, error) {
func NewNNSResolver(address string, settings Settings) (*Resolver, error) {
if address == "" {
return nil, fmt.Errorf("rpc address must not be empty for NNS resolver")
}
if settings == nil {
return nil, fmt.Errorf("resolver settings must not be nil for NNS resolver")
}
var nns ns.NNS
@ -185,10 +212,15 @@ func NewNNSResolver(address string) (*Resolver, error) {
return nil, fmt.Errorf("dial %s: %w", address, err)
}
resolveFunc := func(_ context.Context, name string) (cid.ID, error) {
resolveFunc := func(ctx context.Context, name string) (cid.ID, error) {
var d container.Domain
d.SetName(name)
reqInfo := middleware.GetReqInfo(ctx)
if !slices.Contains(settings.DefaultNamespaces(), reqInfo.Namespace) {
d.SetZone(reqInfo.Namespace + nsDomain)
}
cnrID, err := nns.ResolveContainerDomain(d)
if err != nil {
return cid.ID{}, fmt.Errorf("couldn't resolve container '%s': %w", name, err)

4
api/router.go
View file

@ -96,6 +96,8 @@ type Config struct {
Log *zap.Logger
Metrics *metrics.AppMetrics
RequestMiddlewareSettings s3middleware.RequestSettings
// Domains optional. If empty no virtual hosted domains will be attached.
Domains []string
@ -106,7 +108,7 @@ type Config struct {
func NewRouter(cfg Config) *chi.Mux {
api := chi.NewRouter()
api.Use(
s3middleware.Request(cfg.Log),
s3middleware.Request(cfg.Log, cfg.RequestMiddlewareSettings),
middleware.ThrottleWithOpts(cfg.Throttle),
middleware.Recoverer,
s3middleware.Tracing(),

6
api/router_mock_test.go
View file

@ -18,6 +18,12 @@ func (c *centerMock) Authenticate(*http.Request) (*middleware.Box, error) {
return &middleware.Box{}, nil
}
type requestSettingsMock struct{}
func (r *requestSettingsMock) NamespaceHeader() string {
return "X-Frostfs-Namespace"
}
type handlerMock struct {
t *testing.T
}

1
api/router_test.go
View file

@ -121,6 +121,7 @@ func prepareRouter(t *testing.T) *chi.Mux {
Center: &centerMock{},
Log: zaptest.NewLogger(t),
Metrics: &metrics.AppMetrics{},
RequestMiddlewareSettings: &requestSettingsMock{},
}
return NewRouter(cfg)
}

38
cmd/s3-gw/app.go
View file

@ -10,6 +10,7 @@ import (
"os"
"os/signal"
"runtime/debug"
"strings"
"sync"
"syscall"
"time"
@ -89,6 +90,8 @@ type (
clientCut bool
maxBufferSizeForPut uint64
md5Enabled bool
namespaceHeader string
defaultNamespaces []string
}
maxClientsConfig struct {
@ -195,6 +198,8 @@ func newAppSettings(log *Logger, v *viper.Viper) *appSettings {
settings.initPlacementPolicy(log.logger, v)
settings.setBufferMaxSizeForPut(v.GetUint64(cfgBufferMaxSizeForPut))
settings.setMD5Enabled(v.GetBool(cfgMD5Enabled))
settings.setNamespaceHeader(v.GetString(cfgResolveNamespaceHeader))
settings.setDefaultNamespaces(v.GetStringSlice(cfgKludgeDefaultNamespaces))
return settings
}
@ -324,6 +329,34 @@ func (s *appSettings) setMD5Enabled(md5Enabled bool) {
s.mu.Unlock()
}
func (s *appSettings) NamespaceHeader() string {
s.mu.RLock()
defer s.mu.RUnlock()
return s.namespaceHeader
}
func (s *appSettings) setNamespaceHeader(nsHeader string) {
s.mu.Lock()
s.namespaceHeader = nsHeader
s.mu.Unlock()
}
func (s *appSettings) DefaultNamespaces() []string {
s.mu.RLock()
defer s.mu.RUnlock()
return s.defaultNamespaces
}
func (s *appSettings) setDefaultNamespaces(namespaces []string) {
for i := range namespaces { // to be set namespaces in evn variable as `S3_GW_KLUDGE_DEFAULT_NAMESPACES="" "root"`
namespaces[i] = strings.Trim(namespaces[i], "\"")
}
s.mu.Lock()
s.defaultNamespaces = namespaces
s.mu.Unlock()
}
func (a *App) initAPI(ctx context.Context) {
a.initLayer(ctx)
a.initHandler()
@ -362,6 +395,7 @@ func (a *App) getResolverConfig() *resolver.Config {
return &resolver.Config{
FrostFS: frostfs.NewResolverFrostFS(a.pool),
RPCAddress: a.cfg.GetString(cfgRPCEndpoint),
Settings: a.settings,
}
}
@ -542,6 +576,8 @@ func (a *App) Serve(ctx context.Context) {
Log: a.log,
Metrics: a.metrics,
Domains: domains,
RequestMiddlewareSettings: a.settings,
}
// We cannot make direct assignment if frostfsid.FrostFSID is nil
@ -651,6 +687,8 @@ func (a *App) updateSettings() {
a.settings.setClientCut(a.cfg.GetBool(cfgClientCut))
a.settings.setBufferMaxSizeForPut(a.cfg.GetUint64(cfgBufferMaxSizeForPut))
a.settings.setMD5Enabled(a.cfg.GetBool(cfgMD5Enabled))
a.settings.setNamespaceHeader(a.cfg.GetString(cfgResolveNamespaceHeader))
a.settings.setDefaultNamespaces(a.cfg.GetStringSlice(cfgKludgeDefaultNamespaces))
}
func (a *App) startServices() {

8
cmd/s3-gw/app_settings.go
View file

@ -50,6 +50,8 @@ const (
defaultReadHeaderTimeout = 30 * time.Second
defaultIdleTimeout = 30 * time.Second
defaultNamespaceHeader = "X-Frostfs-Namespace"
)
var defaultCopiesNumbers = []uint32{0}
@ -143,6 +145,7 @@ const ( // Settings.
// Kludge.
cfgKludgeUseDefaultXMLNS = "kludge.use_default_xmlns"
cfgKludgeBypassContentEncodingCheckInChunks = "kludge.bypass_content_encoding_check_in_chunks"
cfgKludgeDefaultNamespaces = "kludge.default_namespaces"
// Web.
cfgWebReadTimeout = "web.read_timeout"
@ -172,6 +175,7 @@ const ( // Settings.
cfgAllowedAccessKeyIDPrefixes = "allowed_access_key_id_prefixes"
// Bucket resolving options.
cfgResolveNamespaceHeader = "resolve_bucket.namespace_header"
cfgResolveBucketAllow = "resolve_bucket.allow"
cfgResolveBucketDeny = "resolve_bucket.deny"
@ -560,6 +564,7 @@ func newSettings() *viper.Viper {
// kludge
v.SetDefault(cfgKludgeUseDefaultXMLNS, false)
v.SetDefault(cfgKludgeBypassContentEncodingCheckInChunks, false)
v.SetDefault(cfgKludgeDefaultNamespaces, []string{"", "root"})
// web
v.SetDefault(cfgWebReadHeaderTimeout, defaultReadHeaderTimeout)
@ -569,6 +574,9 @@ func newSettings() *viper.Viper {
v.SetDefault(cfgFrostfsIDContract, "frostfsid.frostfs")
v.SetDefault(cfgFrostfsIDEnabled, true)
// resolve
v.SetDefault(cfgResolveNamespaceHeader, defaultNamespaceHeader)
// Bind flags
if err := bindFlags(v, flags); err != nil {
panic(fmt.Errorf("bind flags: %w", err))

6
config/config.env
View file

@ -134,6 +134,8 @@ S3_GW_FROSTFS_BUFFER_MAX_SIZE_FOR_PUT=1048576
# If not set, S3 GW will accept all AccessKeyIDs
S3_GW_ALLOWED_ACCESS_KEY_ID_PREFIXES=Ck9BHsgKcnwfCTUSFm6pxhoNS4cBqgN2NQ8zVgPjqZDX 3stjWenX15YwYzczMr88gy3CQr4NYFBQ8P7keGzH5QFn
# Header to determine zone to resolve bucket name
S3_GW_RESOLVE_NAMESPACE_HEADER=X-Frostfs-Namespace
# List of container NNS zones which are allowed or restricted to resolve with HEAD request
S3_GW_RESOLVE_BUCKET_ALLOW=container
# S3_GW_RESOLVE_BUCKET_DENY=
@ -141,7 +143,9 @@ S3_GW_RESOLVE_BUCKET_ALLOW=container
# Enable using default xml namespace `http://s3.amazonaws.com/doc/2006-03-01/` when parse xml bodies.
S3_GW_KLUDGE_USE_DEFAULT_XMLNS=false
# Use this flag to be able to use chunked upload approach without having `aws-chunked` value in `Content-Encoding` header.
S3_GW_BYPASS_CONTENT_ENCODING_CHECK_IN_CHUNKS=false
S3_GW_KLUDGE_BYPASS_CONTENT_ENCODING_CHECK_IN_CHUNKS=false
# Namespaces that should be handled as default
S3_GW_KLUDGE_DEFAULT_NAMESPACES="" "root"
S3_GW_TRACING_ENABLED=false
S3_GW_TRACING_ENDPOINT="localhost:4318"

3
config/config.yaml
View file

@ -163,6 +163,7 @@ allowed_access_key_id_prefixes:
- 3stjWenX15YwYzczMr88gy3CQr4NYFBQ8P7keGzH5QFn
resolve_bucket:
namespace_header: X-Frostfs-Namespace
allow:
- container
deny:
@ -172,6 +173,8 @@ kludge:
use_default_xmlns: false
# Use this flag to be able to use chunked upload approach without having `aws-chunked` value in `Content-Encoding` header.
bypass_content_encoding_check_in_chunks: false
# Namespaces that should be handled as default
default_namespaces: [ "", "root" ]
runtime:
soft_memory_limit: 1gb

16
docs/configuration.md
View file

@ -525,19 +525,21 @@ frostfs:
# `resolve_bucket` section
Bucket name resolving parameters from and to container ID with `HEAD` request.
Bucket name resolving parameters from and to container ID.
```yaml
resolve_bucket:
namespace_header: X-Frostfs-Namespace
allow:
- container
deny:
```
| Parameter | Type | Default value | Description |
|-----------|------------|---------------|--------------------------------------------------------------------------------------------------------------------------|
| `allow` | `[]string` | | List of container zones which are available to resolve. Mutual exclusive with `deny` list. Prioritized over `deny` list. |
| `deny` | `[]string` | | List of container zones which are restricted to resolve. Mutual exclusive with `allow` list. |
| Parameter | Type | SIGHUP reload | Default value | Description |
|--------------------|------------|---------------|-----------------------|--------------------------------------------------------------------------------------------------------------------------|
| `namespace_header` | `string` | yes | `X-Frostfs-Namespace` | Header to determine zone to resolve bucket name. |
| `allow` | `[]string` | no | | List of container zones which are available to resolve. Mutual exclusive with `deny` list. Prioritized over `deny` list. |
| `deny` | `[]string` | no | | List of container zones which are restricted to resolve. Mutual exclusive with `allow` list. |
# `kludge` section
@ -547,12 +549,14 @@ Workarounds for non-standard use cases.
kludge:
use_default_xmlns: false
bypass_content_encoding_check_in_chunks: false
default_namespaces: [ "", "root" ]
```
| Parameter | Type | SIGHUP reload | Default value | Description |
|-------------------------------------------|------------|---------------|---------------|---------------------------------------------------------------------------------------------------------------------------------|
|-------------------------------------------|------------|---------------|---------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
| `use_default_xmlns` | `bool` | yes | false | Enable using default xml namespace `http://s3.amazonaws.com/doc/2006-03-01/` when parse xml bodies. |
| `bypass_content_encoding_check_in_chunks` | `bool` | yes | false | Use this flag to be able to use [chunked upload approach](https://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-streaming.html) without having `aws-chunked` value in `Content-Encoding` header. |
| `default_namespaces` | `[]string` | n/d | ["","root"] | Namespaces that should be handled as default. |
# `runtime` section
Contains runtime parameters.

2
go.mod
View file

@ -28,6 +28,7 @@ require (
go.opentelemetry.io/otel/trace v1.16.0
go.uber.org/zap v1.26.0
golang.org/x/crypto v0.14.0
golang.org/x/exp v0.0.0-20230817173708-d852ddb80c63
google.golang.org/grpc v1.57.0
google.golang.org/protobuf v1.31.0
)
@ -84,7 +85,6 @@ require (
go.opentelemetry.io/otel/sdk v1.16.0 // indirect
go.opentelemetry.io/proto/otlp v0.19.0 // indirect
go.uber.org/multierr v1.11.0 // indirect
golang.org/x/exp v0.0.0-20230817173708-d852ddb80c63 // indirect
golang.org/x/net v0.17.0 // indirect
golang.org/x/sync v0.3.0 // indirect
golang.org/x/sys v0.13.0 // indirect