[#260] Use namespace as domain when resolve bucket

Signed-off-by: Denis Kirillov <d.kirillov@yadro.com>
This commit is contained in:
Denis Kirillov 2023-11-16 15:10:51 +03:00
parent a61ff3b8cb
commit 055cc6a22a
16 changed files with 159 additions and 46 deletions

11
api/cache/buckets.go vendored
View file

@ -39,7 +39,8 @@ func NewBucketCache(config *Config) *BucketCache {
} }
// Get returns a cached object. // Get returns a cached object.
func (o *BucketCache) Get(key string) *data.BucketInfo { func (o *BucketCache) Get(ns, bktName string) *data.BucketInfo {
key := ns + "/" + bktName
entry, err := o.cache.Get(key) entry, err := o.cache.Get(key)
if err != nil { if err != nil {
return nil return nil
@ -56,11 +57,11 @@ func (o *BucketCache) Get(key string) *data.BucketInfo {
} }
// Put puts an object to cache. // Put puts an object to cache.
func (o *BucketCache) Put(bkt *data.BucketInfo) error { func (o *BucketCache) Put(ns string, bkt *data.BucketInfo) error {
return o.cache.Set(bkt.Name, bkt) return o.cache.Set(ns+"/"+bkt.Name, bkt)
} }
// Delete deletes an object from cache. // Delete deletes an object from cache.
func (o *BucketCache) Delete(key string) bool { func (o *BucketCache) Delete(ns, bktName string) bool {
return o.cache.Remove(key) return o.cache.Remove(ns + "/" + bktName)
} }

View file

@ -36,15 +36,15 @@ func TestBucketsCacheType(t *testing.T) {
bktInfo := &data.BucketInfo{Name: "bucket"} bktInfo := &data.BucketInfo{Name: "bucket"}
err := cache.Put(bktInfo) err := cache.Put("", bktInfo)
require.NoError(t, err) require.NoError(t, err)
val := cache.Get(bktInfo.Name) val := cache.Get("", bktInfo.Name)
require.Equal(t, bktInfo, val) require.Equal(t, bktInfo, val)
require.Equal(t, 0, observedLog.Len()) require.Equal(t, 0, observedLog.Len())
err = cache.cache.Set(bktInfo.Name, "tmp") err = cache.cache.Set("/"+bktInfo.Name, "tmp")
require.NoError(t, err) require.NoError(t, err)
assertInvalidCacheEntry(t, cache.Get(bktInfo.Name), observedLog) assertInvalidCacheEntry(t, cache.Get("", bktInfo.Name), observedLog)
} }
func TestObjectNamesCacheType(t *testing.T) { func TestObjectNamesCacheType(t *testing.T) {

View file

@ -56,21 +56,22 @@ func NewCache(cfg *CachesConfig) *Cache {
} }
} }
func (c *Cache) GetBucket(name string) *data.BucketInfo { func (c *Cache) GetBucket(ns, name string) *data.BucketInfo {
return c.bucketCache.Get(name) return c.bucketCache.Get(ns, name)
} }
func (c *Cache) PutBucket(bktInfo *data.BucketInfo) { func (c *Cache) PutBucket(ns string, bktInfo *data.BucketInfo) {
if err := c.bucketCache.Put(bktInfo); err != nil { if err := c.bucketCache.Put(ns, bktInfo); err != nil {
c.logger.Warn(logs.CouldntPutBucketInfoIntoCache, c.logger.Warn(logs.CouldntPutBucketInfoIntoCache,
zap.String("namespace", ns),
zap.String("bucket name", bktInfo.Name), zap.String("bucket name", bktInfo.Name),
zap.Stringer("bucket cid", bktInfo.CID), zap.Stringer("bucket cid", bktInfo.CID),
zap.Error(err)) zap.Error(err))
} }
} }
func (c *Cache) DeleteBucket(name string) { func (c *Cache) DeleteBucket(ns, name string) {
c.bucketCache.Delete(name) c.bucketCache.Delete(ns, name)
} }
func (c *Cache) CleanListCacheEntriesContainingObject(objectName string, cnrID cid.ID) { func (c *Cache) CleanListCacheEntriesContainingObject(objectName string, cnrID cid.ID) {

View file

@ -9,6 +9,7 @@ import (
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api" "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data" "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
s3errors "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors" s3errors "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/middleware"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container"
@ -41,6 +42,8 @@ func (n *layer) containerInfo(ctx context.Context, idCnr cid.ID) (*data.BucketIn
CID: idCnr, CID: idCnr,
Name: idCnr.EncodeToString(), Name: idCnr.EncodeToString(),
} }
reqInfo = middleware.GetReqInfo(ctx)
) )
res, err = n.frostFS.Container(ctx, idCnr) res, err = n.frostFS.Container(ctx, idCnr)
if err != nil { if err != nil {
@ -72,7 +75,7 @@ func (n *layer) containerInfo(ctx context.Context, idCnr cid.ID) (*data.BucketIn
} }
} }
n.cache.PutBucket(info) n.cache.PutBucket(reqInfo.Namespace, info)
return info, nil return info, nil
} }
@ -142,7 +145,7 @@ func (n *layer) createContainer(ctx context.Context, p *CreateBucketParams) (*da
return nil, fmt.Errorf("set container eacl: %w", err) return nil, fmt.Errorf("set container eacl: %w", err)
} }
n.cache.PutBucket(bktInfo) n.cache.PutBucket(bktInfo.Zone, bktInfo)
return bktInfo, nil return bktInfo, nil
} }

View file

@ -401,7 +401,9 @@ func (n *layer) GetBucketInfo(ctx context.Context, name string) (*data.BucketInf
return nil, fmt.Errorf("unescape bucket name: %w", err) return nil, fmt.Errorf("unescape bucket name: %w", err)
} }
if bktInfo := n.cache.GetBucket(name); bktInfo != nil { reqInfo := middleware.GetReqInfo(ctx)
if bktInfo := n.cache.GetBucket(reqInfo.Namespace, name); bktInfo != nil {
return bktInfo, nil return bktInfo, nil
} }
@ -814,6 +816,8 @@ func (n *layer) DeleteBucket(ctx context.Context, p *DeleteBucketParams) error {
return errors.GetAPIError(errors.ErrBucketNotEmpty) return errors.GetAPIError(errors.ErrBucketNotEmpty)
} }
n.cache.DeleteBucket(p.BktInfo.Name) reqInfo := middleware.GetReqInfo(ctx)
n.cache.DeleteBucket(reqInfo.Namespace, p.BktInfo.Name)
return n.frostFS.DeleteContainer(ctx, p.BktInfo.CID, p.SessionToken) return n.frostFS.DeleteContainer(ctx, p.BktInfo.CID, p.SessionToken)
} }

View file

@ -37,6 +37,7 @@ type (
ObjectName string // Object name ObjectName string // Object name
TraceID string // Trace ID TraceID string // Trace ID
URL *url.URL // Request url URL *url.URL // Request url
Namespace string
tags []KeyVal // Any additional info not accommodated by above fields tags []KeyVal // Any additional info not accommodated by above fields
} }
@ -186,7 +187,11 @@ func GetReqLog(ctx context.Context) *zap.Logger {
return nil return nil
} }
func Request(log *zap.Logger) Func { type RequestSettings interface {
NamespaceHeader() string
}
func Request(log *zap.Logger, settings RequestSettings) Func {
return func(h http.Handler) http.Handler { return func(h http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
// generate random UUIDv4 // generate random UUIDv4
@ -200,6 +205,7 @@ func Request(log *zap.Logger) Func {
// set request info into context // set request info into context
// bucket name and object will be set in reqInfo later (limitation of go-chi) // bucket name and object will be set in reqInfo later (limitation of go-chi)
reqInfo := NewReqInfo(w, r, ObjectRequest{}) reqInfo := NewReqInfo(w, r, ObjectRequest{})
reqInfo.Namespace = r.Header.Get(settings.NamespaceHeader())
r = r.WithContext(SetReqInfo(r.Context(), reqInfo)) r = r.WithContext(SetReqInfo(r.Context(), reqInfo))
// set request id into gRPC meta header // set request id into gRPC meta header

View file

@ -6,9 +6,11 @@ import (
"fmt" "fmt"
"sync" "sync"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/middleware"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container"
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/ns" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/ns"
"golang.org/x/exp/slices"
) )
const ( const (
@ -16,6 +18,8 @@ const (
DNSResolver = "dns" DNSResolver = "dns"
) )
const nsDomain = ".ns"
// ErrNoResolvers returns when trying to resolve container without any resolver. // ErrNoResolvers returns when trying to resolve container without any resolver.
var ErrNoResolvers = errors.New("no resolvers") var ErrNoResolvers = errors.New("no resolvers")
@ -28,14 +32,20 @@ type FrostFS interface {
SystemDNS(context.Context) (string, error) SystemDNS(context.Context) (string, error)
} }
type Settings interface {
DefaultNamespaces() []string
}
type Config struct { type Config struct {
FrostFS FrostFS FrostFS FrostFS
RPCAddress string RPCAddress string
Settings Settings
} }
type BucketResolver struct { type BucketResolver struct {
rpcAddress string rpcAddress string
frostfs FrostFS frostfs FrostFS
settings Settings
mu sync.RWMutex mu sync.RWMutex
resolvers []*Resolver resolvers []*Resolver
@ -113,7 +123,13 @@ func (r *BucketResolver) UpdateResolvers(resolverNames []string) error {
return nil return nil
} }
resolvers, err := createResolvers(resolverNames, &Config{FrostFS: r.frostfs, RPCAddress: r.rpcAddress}) cfg := &Config{
FrostFS: r.frostfs,
RPCAddress: r.rpcAddress,
Settings: r.settings,
}
resolvers, err := createResolvers(resolverNames, cfg)
if err != nil { if err != nil {
return err return err
} }
@ -139,25 +155,33 @@ func (r *BucketResolver) equals(resolverNames []string) bool {
func newResolver(name string, cfg *Config) (*Resolver, error) { func newResolver(name string, cfg *Config) (*Resolver, error) {
switch name { switch name {
case DNSResolver: case DNSResolver:
return NewDNSResolver(cfg.FrostFS) return NewDNSResolver(cfg.FrostFS, cfg.Settings)
case NNSResolver: case NNSResolver:
return NewNNSResolver(cfg.RPCAddress) return NewNNSResolver(cfg.RPCAddress, cfg.Settings)
default: default:
return nil, fmt.Errorf("unknown resolver: %s", name) return nil, fmt.Errorf("unknown resolver: %s", name)
} }
} }
func NewDNSResolver(frostFS FrostFS) (*Resolver, error) { func NewDNSResolver(frostFS FrostFS, settings Settings) (*Resolver, error) {
if frostFS == nil { if frostFS == nil {
return nil, fmt.Errorf("pool must not be nil for DNS resolver") return nil, fmt.Errorf("pool must not be nil for DNS resolver")
} }
if settings == nil {
return nil, fmt.Errorf("resolver settings must not be nil for DNS resolver")
}
var dns ns.DNS var dns ns.DNS
resolveFunc := func(ctx context.Context, name string) (cid.ID, error) { resolveFunc := func(ctx context.Context, name string) (cid.ID, error) {
domain, err := frostFS.SystemDNS(ctx) var err error
if err != nil { reqInfo := middleware.GetReqInfo(ctx)
return cid.ID{}, fmt.Errorf("read system DNS parameter of the FrostFS: %w", err) domain := reqInfo.Namespace + nsDomain
if slices.Contains(settings.DefaultNamespaces(), domain) {
domain, err = frostFS.SystemDNS(ctx)
if err != nil {
return cid.ID{}, fmt.Errorf("read system DNS parameter of the FrostFS: %w", err)
}
} }
domain = name + "." + domain domain = name + "." + domain
@ -174,10 +198,13 @@ func NewDNSResolver(frostFS FrostFS) (*Resolver, error) {
}, nil }, nil
} }
func NewNNSResolver(address string) (*Resolver, error) { func NewNNSResolver(address string, settings Settings) (*Resolver, error) {
if address == "" { if address == "" {
return nil, fmt.Errorf("rpc address must not be empty for NNS resolver") return nil, fmt.Errorf("rpc address must not be empty for NNS resolver")
} }
if settings == nil {
return nil, fmt.Errorf("resolver settings must not be nil for NNS resolver")
}
var nns ns.NNS var nns ns.NNS
@ -185,10 +212,15 @@ func NewNNSResolver(address string) (*Resolver, error) {
return nil, fmt.Errorf("dial %s: %w", address, err) return nil, fmt.Errorf("dial %s: %w", address, err)
} }
resolveFunc := func(_ context.Context, name string) (cid.ID, error) { resolveFunc := func(ctx context.Context, name string) (cid.ID, error) {
var d container.Domain var d container.Domain
d.SetName(name) d.SetName(name)
reqInfo := middleware.GetReqInfo(ctx)
if !slices.Contains(settings.DefaultNamespaces(), reqInfo.Namespace) {
d.SetZone(reqInfo.Namespace + nsDomain)
}
cnrID, err := nns.ResolveContainerDomain(d) cnrID, err := nns.ResolveContainerDomain(d)
if err != nil { if err != nil {
return cid.ID{}, fmt.Errorf("couldn't resolve container '%s': %w", name, err) return cid.ID{}, fmt.Errorf("couldn't resolve container '%s': %w", name, err)

View file

@ -96,6 +96,8 @@ type Config struct {
Log *zap.Logger Log *zap.Logger
Metrics *metrics.AppMetrics Metrics *metrics.AppMetrics
RequestMiddlewareSettings s3middleware.RequestSettings
// Domains optional. If empty no virtual hosted domains will be attached. // Domains optional. If empty no virtual hosted domains will be attached.
Domains []string Domains []string
@ -106,7 +108,7 @@ type Config struct {
func NewRouter(cfg Config) *chi.Mux { func NewRouter(cfg Config) *chi.Mux {
api := chi.NewRouter() api := chi.NewRouter()
api.Use( api.Use(
s3middleware.Request(cfg.Log), s3middleware.Request(cfg.Log, cfg.RequestMiddlewareSettings),
middleware.ThrottleWithOpts(cfg.Throttle), middleware.ThrottleWithOpts(cfg.Throttle),
middleware.Recoverer, middleware.Recoverer,
s3middleware.Tracing(), s3middleware.Tracing(),

View file

@ -18,6 +18,12 @@ func (c *centerMock) Authenticate(*http.Request) (*middleware.Box, error) {
return &middleware.Box{}, nil return &middleware.Box{}, nil
} }
type requestSettingsMock struct{}
func (r *requestSettingsMock) NamespaceHeader() string {
return "X-Frostfs-Namespace"
}
type handlerMock struct { type handlerMock struct {
t *testing.T t *testing.T
} }

View file

@ -117,10 +117,11 @@ func prepareRouter(t *testing.T) *chi.Mux {
Limit: 10, Limit: 10,
BacklogTimeout: 30 * time.Second, BacklogTimeout: 30 * time.Second,
}, },
Handler: &handlerMock{t: t}, Handler: &handlerMock{t: t},
Center: &centerMock{}, Center: &centerMock{},
Log: zaptest.NewLogger(t), Log: zaptest.NewLogger(t),
Metrics: &metrics.AppMetrics{}, Metrics: &metrics.AppMetrics{},
RequestMiddlewareSettings: &requestSettingsMock{},
} }
return NewRouter(cfg) return NewRouter(cfg)
} }

View file

@ -10,6 +10,7 @@ import (
"os" "os"
"os/signal" "os/signal"
"runtime/debug" "runtime/debug"
"strings"
"sync" "sync"
"syscall" "syscall"
"time" "time"
@ -89,6 +90,8 @@ type (
clientCut bool clientCut bool
maxBufferSizeForPut uint64 maxBufferSizeForPut uint64
md5Enabled bool md5Enabled bool
namespaceHeader string
defaultNamespaces []string
} }
maxClientsConfig struct { maxClientsConfig struct {
@ -195,6 +198,8 @@ func newAppSettings(log *Logger, v *viper.Viper) *appSettings {
settings.initPlacementPolicy(log.logger, v) settings.initPlacementPolicy(log.logger, v)
settings.setBufferMaxSizeForPut(v.GetUint64(cfgBufferMaxSizeForPut)) settings.setBufferMaxSizeForPut(v.GetUint64(cfgBufferMaxSizeForPut))
settings.setMD5Enabled(v.GetBool(cfgMD5Enabled)) settings.setMD5Enabled(v.GetBool(cfgMD5Enabled))
settings.setNamespaceHeader(v.GetString(cfgResolveNamespaceHeader))
settings.setDefaultNamespaces(v.GetStringSlice(cfgKludgeDefaultNamespaces))
return settings return settings
} }
@ -324,6 +329,34 @@ func (s *appSettings) setMD5Enabled(md5Enabled bool) {
s.mu.Unlock() s.mu.Unlock()
} }
func (s *appSettings) NamespaceHeader() string {
s.mu.RLock()
defer s.mu.RUnlock()
return s.namespaceHeader
}
func (s *appSettings) setNamespaceHeader(nsHeader string) {
s.mu.Lock()
s.namespaceHeader = nsHeader
s.mu.Unlock()
}
func (s *appSettings) DefaultNamespaces() []string {
s.mu.RLock()
defer s.mu.RUnlock()
return s.defaultNamespaces
}
func (s *appSettings) setDefaultNamespaces(namespaces []string) {
for i := range namespaces { // to be set namespaces in evn variable as `S3_GW_KLUDGE_DEFAULT_NAMESPACES="" "root"`
namespaces[i] = strings.Trim(namespaces[i], "\"")
}
s.mu.Lock()
s.defaultNamespaces = namespaces
s.mu.Unlock()
}
func (a *App) initAPI(ctx context.Context) { func (a *App) initAPI(ctx context.Context) {
a.initLayer(ctx) a.initLayer(ctx)
a.initHandler() a.initHandler()
@ -362,6 +395,7 @@ func (a *App) getResolverConfig() *resolver.Config {
return &resolver.Config{ return &resolver.Config{
FrostFS: frostfs.NewResolverFrostFS(a.pool), FrostFS: frostfs.NewResolverFrostFS(a.pool),
RPCAddress: a.cfg.GetString(cfgRPCEndpoint), RPCAddress: a.cfg.GetString(cfgRPCEndpoint),
Settings: a.settings,
} }
} }
@ -542,6 +576,8 @@ func (a *App) Serve(ctx context.Context) {
Log: a.log, Log: a.log,
Metrics: a.metrics, Metrics: a.metrics,
Domains: domains, Domains: domains,
RequestMiddlewareSettings: a.settings,
} }
// We cannot make direct assignment if frostfsid.FrostFSID is nil // We cannot make direct assignment if frostfsid.FrostFSID is nil
@ -651,6 +687,8 @@ func (a *App) updateSettings() {
a.settings.setClientCut(a.cfg.GetBool(cfgClientCut)) a.settings.setClientCut(a.cfg.GetBool(cfgClientCut))
a.settings.setBufferMaxSizeForPut(a.cfg.GetUint64(cfgBufferMaxSizeForPut)) a.settings.setBufferMaxSizeForPut(a.cfg.GetUint64(cfgBufferMaxSizeForPut))
a.settings.setMD5Enabled(a.cfg.GetBool(cfgMD5Enabled)) a.settings.setMD5Enabled(a.cfg.GetBool(cfgMD5Enabled))
a.settings.setNamespaceHeader(a.cfg.GetString(cfgResolveNamespaceHeader))
a.settings.setDefaultNamespaces(a.cfg.GetStringSlice(cfgKludgeDefaultNamespaces))
} }
func (a *App) startServices() { func (a *App) startServices() {

View file

@ -50,6 +50,8 @@ const (
defaultReadHeaderTimeout = 30 * time.Second defaultReadHeaderTimeout = 30 * time.Second
defaultIdleTimeout = 30 * time.Second defaultIdleTimeout = 30 * time.Second
defaultNamespaceHeader = "X-Frostfs-Namespace"
) )
var defaultCopiesNumbers = []uint32{0} var defaultCopiesNumbers = []uint32{0}
@ -143,6 +145,7 @@ const ( // Settings.
// Kludge. // Kludge.
cfgKludgeUseDefaultXMLNS = "kludge.use_default_xmlns" cfgKludgeUseDefaultXMLNS = "kludge.use_default_xmlns"
cfgKludgeBypassContentEncodingCheckInChunks = "kludge.bypass_content_encoding_check_in_chunks" cfgKludgeBypassContentEncodingCheckInChunks = "kludge.bypass_content_encoding_check_in_chunks"
cfgKludgeDefaultNamespaces = "kludge.default_namespaces"
// Web. // Web.
cfgWebReadTimeout = "web.read_timeout" cfgWebReadTimeout = "web.read_timeout"
@ -172,8 +175,9 @@ const ( // Settings.
cfgAllowedAccessKeyIDPrefixes = "allowed_access_key_id_prefixes" cfgAllowedAccessKeyIDPrefixes = "allowed_access_key_id_prefixes"
// Bucket resolving options. // Bucket resolving options.
cfgResolveBucketAllow = "resolve_bucket.allow" cfgResolveNamespaceHeader = "resolve_bucket.namespace_header"
cfgResolveBucketDeny = "resolve_bucket.deny" cfgResolveBucketAllow = "resolve_bucket.allow"
cfgResolveBucketDeny = "resolve_bucket.deny"
// Runtime. // Runtime.
cfgSoftMemoryLimit = "runtime.soft_memory_limit" cfgSoftMemoryLimit = "runtime.soft_memory_limit"
@ -560,6 +564,7 @@ func newSettings() *viper.Viper {
// kludge // kludge
v.SetDefault(cfgKludgeUseDefaultXMLNS, false) v.SetDefault(cfgKludgeUseDefaultXMLNS, false)
v.SetDefault(cfgKludgeBypassContentEncodingCheckInChunks, false) v.SetDefault(cfgKludgeBypassContentEncodingCheckInChunks, false)
v.SetDefault(cfgKludgeDefaultNamespaces, []string{"", "root"})
// web // web
v.SetDefault(cfgWebReadHeaderTimeout, defaultReadHeaderTimeout) v.SetDefault(cfgWebReadHeaderTimeout, defaultReadHeaderTimeout)
@ -569,6 +574,9 @@ func newSettings() *viper.Viper {
v.SetDefault(cfgFrostfsIDContract, "frostfsid.frostfs") v.SetDefault(cfgFrostfsIDContract, "frostfsid.frostfs")
v.SetDefault(cfgFrostfsIDEnabled, true) v.SetDefault(cfgFrostfsIDEnabled, true)
// resolve
v.SetDefault(cfgResolveNamespaceHeader, defaultNamespaceHeader)
// Bind flags // Bind flags
if err := bindFlags(v, flags); err != nil { if err := bindFlags(v, flags); err != nil {
panic(fmt.Errorf("bind flags: %w", err)) panic(fmt.Errorf("bind flags: %w", err))

View file

@ -134,6 +134,8 @@ S3_GW_FROSTFS_BUFFER_MAX_SIZE_FOR_PUT=1048576
# If not set, S3 GW will accept all AccessKeyIDs # If not set, S3 GW will accept all AccessKeyIDs
S3_GW_ALLOWED_ACCESS_KEY_ID_PREFIXES=Ck9BHsgKcnwfCTUSFm6pxhoNS4cBqgN2NQ8zVgPjqZDX 3stjWenX15YwYzczMr88gy3CQr4NYFBQ8P7keGzH5QFn S3_GW_ALLOWED_ACCESS_KEY_ID_PREFIXES=Ck9BHsgKcnwfCTUSFm6pxhoNS4cBqgN2NQ8zVgPjqZDX 3stjWenX15YwYzczMr88gy3CQr4NYFBQ8P7keGzH5QFn
# Header to determine zone to resolve bucket name
S3_GW_RESOLVE_NAMESPACE_HEADER=X-Frostfs-Namespace
# List of container NNS zones which are allowed or restricted to resolve with HEAD request # List of container NNS zones which are allowed or restricted to resolve with HEAD request
S3_GW_RESOLVE_BUCKET_ALLOW=container S3_GW_RESOLVE_BUCKET_ALLOW=container
# S3_GW_RESOLVE_BUCKET_DENY= # S3_GW_RESOLVE_BUCKET_DENY=
@ -141,7 +143,9 @@ S3_GW_RESOLVE_BUCKET_ALLOW=container
# Enable using default xml namespace `http://s3.amazonaws.com/doc/2006-03-01/` when parse xml bodies. # Enable using default xml namespace `http://s3.amazonaws.com/doc/2006-03-01/` when parse xml bodies.
S3_GW_KLUDGE_USE_DEFAULT_XMLNS=false S3_GW_KLUDGE_USE_DEFAULT_XMLNS=false
# Use this flag to be able to use chunked upload approach without having `aws-chunked` value in `Content-Encoding` header. # Use this flag to be able to use chunked upload approach without having `aws-chunked` value in `Content-Encoding` header.
S3_GW_BYPASS_CONTENT_ENCODING_CHECK_IN_CHUNKS=false S3_GW_KLUDGE_BYPASS_CONTENT_ENCODING_CHECK_IN_CHUNKS=false
# Namespaces that should be handled as default
S3_GW_KLUDGE_DEFAULT_NAMESPACES="" "root"
S3_GW_TRACING_ENABLED=false S3_GW_TRACING_ENABLED=false
S3_GW_TRACING_ENDPOINT="localhost:4318" S3_GW_TRACING_ENDPOINT="localhost:4318"

View file

@ -163,6 +163,7 @@ allowed_access_key_id_prefixes:
- 3stjWenX15YwYzczMr88gy3CQr4NYFBQ8P7keGzH5QFn - 3stjWenX15YwYzczMr88gy3CQr4NYFBQ8P7keGzH5QFn
resolve_bucket: resolve_bucket:
namespace_header: X-Frostfs-Namespace
allow: allow:
- container - container
deny: deny:
@ -172,6 +173,8 @@ kludge:
use_default_xmlns: false use_default_xmlns: false
# Use this flag to be able to use chunked upload approach without having `aws-chunked` value in `Content-Encoding` header. # Use this flag to be able to use chunked upload approach without having `aws-chunked` value in `Content-Encoding` header.
bypass_content_encoding_check_in_chunks: false bypass_content_encoding_check_in_chunks: false
# Namespaces that should be handled as default
default_namespaces: [ "", "root" ]
runtime: runtime:
soft_memory_limit: 1gb soft_memory_limit: 1gb

View file

@ -525,19 +525,21 @@ frostfs:
# `resolve_bucket` section # `resolve_bucket` section
Bucket name resolving parameters from and to container ID with `HEAD` request. Bucket name resolving parameters from and to container ID.
```yaml ```yaml
resolve_bucket: resolve_bucket:
namespace_header: X-Frostfs-Namespace
allow: allow:
- container - container
deny: deny:
``` ```
| Parameter | Type | Default value | Description | | Parameter | Type | SIGHUP reload | Default value | Description |
|-----------|------------|---------------|--------------------------------------------------------------------------------------------------------------------------| |--------------------|------------|---------------|-----------------------|--------------------------------------------------------------------------------------------------------------------------|
| `allow` | `[]string` | | List of container zones which are available to resolve. Mutual exclusive with `deny` list. Prioritized over `deny` list. | | `namespace_header` | `string` | yes | `X-Frostfs-Namespace` | Header to determine zone to resolve bucket name. |
| `deny` | `[]string` | | List of container zones which are restricted to resolve. Mutual exclusive with `allow` list. | | `allow` | `[]string` | no | | List of container zones which are available to resolve. Mutual exclusive with `deny` list. Prioritized over `deny` list. |
| `deny` | `[]string` | no | | List of container zones which are restricted to resolve. Mutual exclusive with `allow` list. |
# `kludge` section # `kludge` section
@ -547,12 +549,14 @@ Workarounds for non-standard use cases.
kludge: kludge:
use_default_xmlns: false use_default_xmlns: false
bypass_content_encoding_check_in_chunks: false bypass_content_encoding_check_in_chunks: false
default_namespaces: [ "", "root" ]
``` ```
| Parameter | Type | SIGHUP reload | Default value | Description | | Parameter | Type | SIGHUP reload | Default value | Description |
|-------------------------------------------|------------|---------------|---------------|---------------------------------------------------------------------------------------------------------------------------------| |-------------------------------------------|------------|---------------|---------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
| `use_default_xmlns` | `bool` | yes | false | Enable using default xml namespace `http://s3.amazonaws.com/doc/2006-03-01/` when parse xml bodies. | | `use_default_xmlns` | `bool` | yes | false | Enable using default xml namespace `http://s3.amazonaws.com/doc/2006-03-01/` when parse xml bodies. |
| `bypass_content_encoding_check_in_chunks` | `bool` | yes | false | Use this flag to be able to use [chunked upload approach](https://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-streaming.html) without having `aws-chunked` value in `Content-Encoding` header. | | `bypass_content_encoding_check_in_chunks` | `bool` | yes | false | Use this flag to be able to use [chunked upload approach](https://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-streaming.html) without having `aws-chunked` value in `Content-Encoding` header. |
| `default_namespaces` | `[]string` | n/d | ["","root"] | Namespaces that should be handled as default. |
# `runtime` section # `runtime` section
Contains runtime parameters. Contains runtime parameters.

2
go.mod
View file

@ -28,6 +28,7 @@ require (
go.opentelemetry.io/otel/trace v1.16.0 go.opentelemetry.io/otel/trace v1.16.0
go.uber.org/zap v1.26.0 go.uber.org/zap v1.26.0
golang.org/x/crypto v0.14.0 golang.org/x/crypto v0.14.0
golang.org/x/exp v0.0.0-20230817173708-d852ddb80c63
google.golang.org/grpc v1.57.0 google.golang.org/grpc v1.57.0
google.golang.org/protobuf v1.31.0 google.golang.org/protobuf v1.31.0
) )
@ -84,7 +85,6 @@ require (
go.opentelemetry.io/otel/sdk v1.16.0 // indirect go.opentelemetry.io/otel/sdk v1.16.0 // indirect
go.opentelemetry.io/proto/otlp v0.19.0 // indirect go.opentelemetry.io/proto/otlp v0.19.0 // indirect
go.uber.org/multierr v1.11.0 // indirect go.uber.org/multierr v1.11.0 // indirect
golang.org/x/exp v0.0.0-20230817173708-d852ddb80c63 // indirect
golang.org/x/net v0.17.0 // indirect golang.org/x/net v0.17.0 // indirect
golang.org/x/sync v0.3.0 // indirect golang.org/x/sync v0.3.0 // indirect
golang.org/x/sys v0.13.0 // indirect golang.org/x/sys v0.13.0 // indirect