TrueCloudLab/frostfs-s3-gw
17
3
Fork 19
Code Issues 30 Pull requests 8 Projects Releases 18 Packages Activity Actions

feature/namespaced_placement_policy_config #266

Merged
alexvanin merged 1 commit from dkirillov/frostfs-s3-gw:feature/namespaced_placement_policy_config into master 2023-11-30 14:27:09 +00:00
Conversation 2 Commits 1 Files changed 18 +307 -52
18 changed files with 307 additions and 52 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
CHANGELOG.md
View file

@ -40,6 +40,7 @@ This document outlines major changes between releases.
- Add new `logger.destination` config param (#236)
- Add `X-Amz-Content-Sha256` header validation (#218)
- Support frostfsid contract. See `frostfsid` config section (#260)
- Support per namespace placement policies configuration (see `namespaces.config` config param) (#266)
### Changed
- Update prometheus to v1.15.0 (#94)

14
api/handler/api.go
View file

@ -31,10 +31,10 @@ type (
// Config contains data which handler needs to keep.
Config interface {
DefaultPlacementPolicy() netmap.PlacementPolicy
PlacementPolicy(string) (netmap.PlacementPolicy, bool)
CopiesNumbers(string) ([]uint32, bool)
DefaultCopiesNumbers() []uint32
DefaultPlacementPolicy(namespace string) netmap.PlacementPolicy
PlacementPolicy(namespace, constraint string) (netmap.PlacementPolicy, bool)
CopiesNumbers(namespace, constraint string) ([]uint32, bool)
DefaultCopiesNumbers(namespace string) []uint32
NewXMLDecoder(io.Reader) *xml.Decoder
DefaultMaxAge() int
NotificatorEnabled() bool
@ -74,7 +74,7 @@ func New(log *zap.Logger, obj layer.Client, notificator Notificator, cfg Config)
// 1) array of copies numbers sent in request's header has the highest priority.
// 2) array of copies numbers with corresponding location constraint provided in the config file.
// 3) default copies number from the config file wrapped into array.
func (h *handler) pickCopiesNumbers(metadata map[string]string, locationConstraint string) ([]uint32, error) {
func (h *handler) pickCopiesNumbers(metadata map[string]string, namespace, locationConstraint string) ([]uint32, error) {
copiesNumbersStr, ok := metadata[layer.AttributeFrostfsCopiesNumber]
if ok {
result, err := parseCopiesNumbers(copiesNumbersStr)
@ -84,12 +84,12 @@ func (h *handler) pickCopiesNumbers(metadata map[string]string, locationConstrai
return result, nil
}
copiesNumbers, ok := h.cfg.CopiesNumbers(locationConstraint)
copiesNumbers, ok := h.cfg.CopiesNumbers(namespace, locationConstraint)
if ok {
return copiesNumbers, nil
}
return h.cfg.DefaultCopiesNumbers(), nil
return h.cfg.DefaultCopiesNumbers(namespace), nil
}
func parseCopiesNumbers(copiesNumbersStr string) ([]uint32, error) {

10
api/handler/api_test.go
View file

@ -26,7 +26,7 @@ func TestCopiesNumberPicker(t *testing.T) {
metadata["somekey1"] = "5, 6, 7"
expectedCopiesNumbers := []uint32{1}
actualCopiesNumbers, err := h.pickCopiesNumbers(metadata, locationConstraint2)
actualCopiesNumbers, err := h.pickCopiesNumbers(metadata, "", locationConstraint2)
require.NoError(t, err)
require.Equal(t, expectedCopiesNumbers, actualCopiesNumbers)
})
@ -35,7 +35,7 @@ func TestCopiesNumberPicker(t *testing.T) {
metadata["somekey2"] = "6, 7, 8"
expectedCopiesNumbers := []uint32{2, 3, 4}
actualCopiesNumbers, err := h.pickCopiesNumbers(metadata, locationConstraint1)
actualCopiesNumbers, err := h.pickCopiesNumbers(metadata, "", locationConstraint1)
require.NoError(t, err)
require.Equal(t, expectedCopiesNumbers, actualCopiesNumbers)
})
@ -44,7 +44,7 @@ func TestCopiesNumberPicker(t *testing.T) {
metadata["frostfs-copies-number"] = "7, 8, 9"
expectedCopiesNumbers := []uint32{7, 8, 9}
actualCopiesNumbers, err := h.pickCopiesNumbers(metadata, locationConstraint2)
actualCopiesNumbers, err := h.pickCopiesNumbers(metadata, "", locationConstraint2)
require.NoError(t, err)
require.Equal(t, expectedCopiesNumbers, actualCopiesNumbers)
})
@ -53,7 +53,7 @@ func TestCopiesNumberPicker(t *testing.T) {
metadata["frostfs-copies-number"] = "7,8,9"
expectedCopiesNumbers := []uint32{7, 8, 9}
actualCopiesNumbers, err := h.pickCopiesNumbers(metadata, locationConstraint2)
actualCopiesNumbers, err := h.pickCopiesNumbers(metadata, "", locationConstraint2)
require.NoError(t, err)
require.Equal(t, expectedCopiesNumbers, actualCopiesNumbers)
})
@ -62,7 +62,7 @@ func TestCopiesNumberPicker(t *testing.T) {
metadata["frostfs-copies-number"] = "11, 12, 13, "
expectedCopiesNumbers := []uint32{11, 12, 13}
actualCopiesNumbers, err := h.pickCopiesNumbers(metadata, locationConstraint2)
actualCopiesNumbers, err := h.pickCopiesNumbers(metadata, "", locationConstraint2)
require.NoError(t, err)
require.Equal(t, expectedCopiesNumbers, actualCopiesNumbers)
})

2
api/handler/copy.go
View file

@ -204,7 +204,7 @@ func (h *handler) CopyObjectHandler(w http.ResponseWriter, r *http.Request) {
DstEncryption: dstEncryptionParams,
}
params.CopiesNumbers, err = h.pickCopiesNumbers(metadata, dstBktInfo.LocationConstraint)
params.CopiesNumbers, err = h.pickCopiesNumbers(metadata, reqInfo.Namespace, dstBktInfo.LocationConstraint)
if err != nil {
h.logAndSendError(w, "invalid copies number", reqInfo, err)
return

2
api/handler/cors.go
View file

@ -55,7 +55,7 @@ func (h *handler) PutBucketCorsHandler(w http.ResponseWriter, r *http.Request) {
NewDecoder: h.cfg.NewXMLDecoder,
}
p.CopiesNumbers, err = h.pickCopiesNumbers(parseMetadata(r), bktInfo.LocationConstraint)
p.CopiesNumbers, err = h.pickCopiesNumbers(parseMetadata(r), reqInfo.Namespace, bktInfo.LocationConstraint)
if err != nil {
h.logAndSendError(w, "invalid copies number", reqInfo, err)
return

8
api/handler/handlers_test.go
View file

@ -67,20 +67,20 @@ type configMock struct {
md5Enabled bool
}
func (c *configMock) DefaultPlacementPolicy() netmap.PlacementPolicy {
func (c *configMock) DefaultPlacementPolicy(_ string) netmap.PlacementPolicy {
return c.defaultPolicy
}
func (c *configMock) PlacementPolicy(string) (netmap.PlacementPolicy, bool) {
func (c *configMock) PlacementPolicy(_, _ string) (netmap.PlacementPolicy, bool) {
return netmap.PlacementPolicy{}, false
}
func (c *configMock) CopiesNumbers(locationConstraint string) ([]uint32, bool) {
func (c *configMock) CopiesNumbers(_, locationConstraint string) ([]uint32, bool) {
result, ok := c.copiesNumbers[locationConstraint]
return result, ok
}
func (c *configMock) DefaultCopiesNumbers() []uint32 {
func (c *configMock) DefaultCopiesNumbers(_ string) []uint32 {
return c.defaultCopiesNumbers
}

4
api/handler/locking.go
View file

@ -145,7 +145,7 @@ func (h *handler) PutObjectLegalHoldHandler(w http.ResponseWriter, r *http.Reque
},
}
p.CopiesNumbers, err = h.pickCopiesNumbers(parseMetadata(r), bktInfo.LocationConstraint)
p.CopiesNumbers, err = h.pickCopiesNumbers(parseMetadata(r), reqInfo.Namespace, bktInfo.LocationConstraint)
if err != nil {
h.logAndSendError(w, "invalid copies number", reqInfo, err)
return
@ -229,7 +229,7 @@ func (h *handler) PutObjectRetentionHandler(w http.ResponseWriter, r *http.Reque
NewLock: lock,
}
p.CopiesNumbers, err = h.pickCopiesNumbers(parseMetadata(r), bktInfo.LocationConstraint)
p.CopiesNumbers, err = h.pickCopiesNumbers(parseMetadata(r), reqInfo.Namespace, bktInfo.LocationConstraint)
if err != nil {
h.logAndSendError(w, "invalid copies number", reqInfo, err)
return

2
api/handler/multipart_upload.go
View file

@ -157,7 +157,7 @@ func (h *handler) CreateMultipartUploadHandler(w http.ResponseWriter, r *http.Re
p.Header[api.ContentLanguage] = contentLanguage
}
p.CopiesNumbers, err = h.pickCopiesNumbers(p.Header, bktInfo.LocationConstraint)
p.CopiesNumbers, err = h.pickCopiesNumbers(p.Header, reqInfo.Namespace, bktInfo.LocationConstraint)
if err != nil {
h.logAndSendError(w, "invalid copies number", reqInfo, err, additional...)
return

2
api/handler/notifications.go
View file

@ -115,7 +115,7 @@ func (h *handler) PutBucketNotificationHandler(w http.ResponseWriter, r *http.Re
Configuration: conf,
}
p.CopiesNumbers, err = h.pickCopiesNumbers(parseMetadata(r), bktInfo.LocationConstraint)
p.CopiesNumbers, err = h.pickCopiesNumbers(parseMetadata(r), reqInfo.Namespace, bktInfo.LocationConstraint)
if err != nil {
h.logAndSendError(w, "invalid copies number", reqInfo, err)
return

10
api/handler/put.go
View file

@ -248,7 +248,7 @@ func (h *handler) PutObjectHandler(w http.ResponseWriter, r *http.Request) {
ContentSHA256Hash: r.Header.Get(api.AmzContentSha256),
}
params.CopiesNumbers, err = h.pickCopiesNumbers(metadata, bktInfo.LocationConstraint)
params.CopiesNumbers, err = h.pickCopiesNumbers(metadata, reqInfo.Namespace, bktInfo.LocationConstraint)
if err != nil {
h.logAndSendError(w, "invalid copies number", reqInfo, err)
return
@ -800,7 +800,7 @@ func (h *handler) CreateBucketHandler(w http.ResponseWriter, r *http.Request) {
return
}
if err = h.setPolicy(p, createParams.LocationConstraint, policies); err != nil {
if err = h.setPolicy(p, reqInfo.Namespace, createParams.LocationConstraint, policies); err != nil {
h.logAndSendError(w, "couldn't set placement policy", reqInfo, err)
return
}
@ -830,8 +830,8 @@ func (h *handler) CreateBucketHandler(w http.ResponseWriter, r *http.Request) {
middleware.WriteSuccessResponseHeadersOnly(w)
}
func (h handler) setPolicy(prm *layer.CreateBucketParams, locationConstraint string, userPolicies []*accessbox.ContainerPolicy) error {
prm.Policy = h.cfg.DefaultPlacementPolicy()
func (h handler) setPolicy(prm *layer.CreateBucketParams, namespace, locationConstraint string, userPolicies []*accessbox.ContainerPolicy) error {
prm.Policy = h.cfg.DefaultPlacementPolicy(namespace)
prm.LocationConstraint = locationConstraint
if locationConstraint == "" {
@ -845,7 +845,7 @@ func (h handler) setPolicy(prm *layer.CreateBucketParams, locationConstraint str
}
}
if policy, ok := h.cfg.PlacementPolicy(locationConstraint); ok {
if policy, ok := h.cfg.PlacementPolicy(namespace, locationConstraint); ok {
prm.Policy = policy
return nil
}

37
cmd/s3-gw/app.go
View file

@ -83,10 +83,7 @@ type (
isResolveListAllow bool // True if ResolveZoneList contains allowed zones
mu sync.RWMutex
defaultPolicy netmap.PlacementPolicy
regionMap map[string]netmap.PlacementPolicy
copiesNumbers map[string][]uint32
defaultCopiesNumbers []uint32
namespaces Namespaces
defaultXMLNS bool
bypassContentEncodingInChunks bool
clientCut bool
@ -187,6 +184,8 @@ func newAppSettings(log *Logger, v *viper.Viper) *appSettings {
defaultXMLNS: v.GetBool(cfgKludgeUseDefaultXMLNS),
defaultMaxAge: fetchDefaultMaxAge(v, log.logger),
notificatorEnabled: v.GetBool(cfgEnableNATS),
defaultNamespaces: fetchDefaultNamespaces(log.logger, v),
namespaceHeader: v.GetString(cfgResolveNamespaceHeader),
}
settings.resolveZoneList = v.GetStringSlice(cfgResolveBucketAllow)
@ -200,8 +199,6 @@ func newAppSettings(log *Logger, v *viper.Viper) *appSettings {
settings.initPlacementPolicy(log.logger, v)
settings.setBufferMaxSizeForPut(v.GetUint64(cfgBufferMaxSizeForPut))
settings.setMD5Enabled(v.GetBool(cfgMD5Enabled))
settings.setNamespaceHeader(v.GetString(cfgResolveNamespaceHeader))
settings.setDefaultNamespaces(v.GetStringSlice(cfgKludgeDefaultNamespaces))
return settings
}
@ -243,46 +240,40 @@ func (s *appSettings) setBufferMaxSizeForPut(size uint64) {
}
func (s *appSettings) initPlacementPolicy(l *zap.Logger, v *viper.Viper) {
defaultPolicy := fetchDefaultPolicy(l, v)
regionMap := fetchRegionMappingPolicies(l, v)
defaultCopies := fetchDefaultCopiesNumbers(l, v)
copiesNumbers := fetchCopiesNumbers(l, v)
nsConfig := fetchNamespacesConfig(l, v)
s.mu.Lock()
defer s.mu.Unlock()
s.defaultPolicy = defaultPolicy
s.regionMap = regionMap
s.defaultCopiesNumbers = defaultCopies
s.copiesNumbers = copiesNumbers
s.namespaces = nsConfig.Namespaces
}
func (s *appSettings) DefaultPlacementPolicy() netmap.PlacementPolicy {
func (s *appSettings) DefaultPlacementPolicy(namespace string) netmap.PlacementPolicy {
s.mu.RLock()
defer s.mu.RUnlock()
return s.defaultPolicy
return s.namespaces[namespace].LocationConstraints[defaultConstraintName]
}
func (s *appSettings) PlacementPolicy(name string) (netmap.PlacementPolicy, bool) {
func (s *appSettings) PlacementPolicy(namespace, constraint string) (netmap.PlacementPolicy, bool) {
s.mu.RLock()
policy, ok := s.regionMap[name]
policy, ok := s.namespaces[namespace].LocationConstraints[constraint]
s.mu.RUnlock()
return policy, ok
}
func (s *appSettings) CopiesNumbers(locationConstraint string) ([]uint32, bool) {
func (s *appSettings) CopiesNumbers(namespace, constraint string) ([]uint32, bool) {
s.mu.RLock()
copiesNumbers, ok := s.copiesNumbers[locationConstraint]
copiesNumbers, ok := s.namespaces[namespace].CopiesNumbers[constraint]
s.mu.RUnlock()
return copiesNumbers, ok
}
func (s *appSettings) DefaultCopiesNumbers() []uint32 {
func (s *appSettings) DefaultCopiesNumbers(namespace string) []uint32 {
s.mu.RLock()
defer s.mu.RUnlock()
return s.defaultCopiesNumbers
return s.namespaces[namespace].CopiesNumbers[defaultConstraintName]
}
func (s *appSettings) NewXMLDecoder(r io.Reader) *xml.Decoder {
@ -687,6 +678,7 @@ func (a *App) updateSettings() {
a.settings.logLevel.SetLevel(lvl)
}
a.settings.setNamespaceHeader(a.cfg.GetString(cfgResolveNamespaceHeader)) // should be updated before placement policies
a.settings.initPlacementPolicy(a.log, a.cfg)
a.settings.useDefaultXMLNamespace(a.cfg.GetBool(cfgKludgeUseDefaultXMLNS))
@ -694,7 +686,6 @@ func (a *App) updateSettings() {
a.settings.setClientCut(a.cfg.GetBool(cfgClientCut))
a.settings.setBufferMaxSizeForPut(a.cfg.GetUint64(cfgBufferMaxSizeForPut))
a.settings.setMD5Enabled(a.cfg.GetBool(cfgMD5Enabled))
a.settings.setNamespaceHeader(a.cfg.GetString(cfgResolveNamespaceHeader))
a.settings.setDefaultNamespaces(a.cfg.GetStringSlice(cfgKludgeDefaultNamespaces))
}

87
cmd/s3-gw/app_settings.go
View file

@ -52,9 +52,14 @@ const (
defaultIdleTimeout = 30 * time.Second
defaultNamespaceHeader = "X-Frostfs-Namespace"
defaultConstraintName = "default"
)
var defaultCopiesNumbers = []uint32{0}
var (
defaultCopiesNumbers = []uint32{0}
defaultDefaultNamespaces = []string{"", "root"}
)
const ( // Settings.
// Logger.
@ -153,6 +158,9 @@ const ( // Settings.
cfgWebWriteTimeout = "web.write_timeout"
cfgWebIdleTimeout = "web.idle_timeout"
// Namespaces.
cfgNamespacesConfig = "namespaces.config"
// Command line args.
cmdHelp = "help"
cmdVersion = "version"
@ -450,6 +458,81 @@ func fetchCopiesNumbers(l *zap.Logger, v *viper.Viper) map[string][]uint32 {
return copiesNums
}
func fetchDefaultNamespaces(l *zap.Logger, v *viper.Viper) []string {
defaultNamespaces := v.GetStringSlice(cfgKludgeDefaultNamespaces)
if len(defaultNamespaces) == 0 {
defaultNamespaces = defaultDefaultNamespaces
l.Warn(logs.DefaultNamespacesCannotBeEmpty, zap.Strings("namespaces", defaultNamespaces))
}
return defaultNamespaces
}
func fetchNamespacesConfig(l *zap.Logger, v *viper.Viper) NamespacesConfig {
defaultNSRegionMap := fetchRegionMappingPolicies(l, v)
defaultNSRegionMap[defaultConstraintName] = fetchDefaultPolicy(l, v)
defaultNSCopiesNumbers := fetchCopiesNumbers(l, v)
defaultNSCopiesNumbers[defaultConstraintName] = fetchDefaultCopiesNumbers(l, v)
defaultNSValue := Namespace{
LocationConstraints: defaultNSRegionMap,
CopiesNumbers: defaultNSCopiesNumbers,
}
nsConfig, err := readNamespacesConfig(v.GetString(cfgNamespacesConfig))
if err != nil {
l.Warn(logs.FailedToParseNamespacesConfig)
}
defaultNamespacesNames := fetchDefaultNamespaces(l, v)
var overrideDefaults []Namespace
for _, name := range defaultNamespacesNames {
if ns, ok := nsConfig.Namespaces[name]; ok {
overrideDefaults = append(overrideDefaults, ns)
delete(nsConfig.Namespaces, name)
}
}
if len(overrideDefaults) > 0 {
l.Warn(logs.DefaultNamespaceConfigValuesBeOverwritten)
defaultNSValue.LocationConstraints = overrideDefaults[0].LocationConstraints
defaultNSValue.CopiesNumbers = overrideDefaults[0].CopiesNumbers
if len(overrideDefaults) > 1 {
l.Warn(logs.MultipleDefaultOverridesFound, zap.String("name", overrideDefaults[0].Name))
}
}
for _, name := range defaultNamespacesNames {
nsConfig.Namespaces[name] = Namespace{
Name: name,
LocationConstraints: defaultNSValue.LocationConstraints,
CopiesNumbers: defaultNSValue.CopiesNumbers,
}
}
return nsConfig
}
func readNamespacesConfig(filepath string) (NamespacesConfig, error) {
if filepath == "" {
return NamespacesConfig{}, nil
}
data, err := os.ReadFile(filepath)
if err != nil {
return NamespacesConfig{}, fmt.Errorf("failed to read namespace config '%s': %w", filepath, err)
}
var nsConfig NamespacesConfig
if err = json.Unmarshal(data, &nsConfig); err != nil {
return NamespacesConfig{}, fmt.Errorf("failed to parse namespace config: %w", err)
}
return nsConfig, nil
}
func fetchPeers(l *zap.Logger, v *viper.Viper) []pool.NodeParam {
var nodes []pool.NodeParam
for i := 0; ; i++ {
@ -564,7 +647,7 @@ func newSettings() *viper.Viper {
// kludge
v.SetDefault(cfgKludgeUseDefaultXMLNS, false)
v.SetDefault(cfgKludgeBypassContentEncodingCheckInChunks, false)
v.SetDefault(cfgKludgeDefaultNamespaces, []string{"", "root"})
v.SetDefault(cfgKludgeDefaultNamespaces, defaultDefaultNamespaces)
// web
v.SetDefault(cfgWebReadHeaderTimeout, defaultReadHeaderTimeout)

51
cmd/s3-gw/decoder_test.go
View file

@ -2,6 +2,7 @@ package main
import (
"bytes"
"encoding/json"
"testing"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/handler"
@ -93,3 +94,53 @@ func TestDefaultNamespace(t *testing.T) {
})
}
}
func TestNamespacesMarshaling(t *testing.T) {
dataJSON := `
{
"namespaces": {
"kapusta": {
"location_constraints": {
"default": "REP 3",
"load-1-1": "REP 1 CBF 1 SELECT 1 FROM *"
},
"copies_numbers": {
"default": [
0
],
"load-1-1": [
1
]
}
},
"root": {
"location_constraints": {
"default": "REP 3",
"test": "{\"replicas\":[{\"count\":1,\"selector\":\"\"}],\"containerBackupFactor\":1,\"selectors\":[{\"name\":\"\",\"count\":1,\"clause\":\"CLAUSE_UNSPECIFIED\",\"attribute\":\"\",\"filter\":\"Color\"}],\"filters\":[{\"name\":\"Color\",\"key\":\"Color\",\"op\":\"EQ\",\"value\":\"Red\",\"filters\":[]}],\"unique\":false}"
},
"copies_numbers": {
"default": [
0
],
"load-1-1": [
1
]
}
}
}
}
`
var nsConfig NamespacesConfig
err := json.Unmarshal([]byte(dataJSON), &nsConfig)
require.NoError(t, err)
data, err := json.Marshal(nsConfig)
require.NoError(t, err)
var nsConfig2 NamespacesConfig
err = json.Unmarshal(data, &nsConfig2)
require.NoError(t, err)
require.Equal(t, nsConfig, nsConfig2)
}

79
cmd/s3-gw/namespaces.go Normal file
View file

@ -0,0 +1,79 @@
package main
import (
"encoding/json"
"fmt"
"strings"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap"
)
type NamespacesConfig struct {
Namespaces Namespaces `json:"namespaces"`
}
type Namespaces map[string]Namespace
type Namespace struct {
Name string `json:"-"`
LocationConstraints LocationConstraints `json:"location_constraints"`
CopiesNumbers map[string][]uint32 `json:"copies_numbers"`
}
type LocationConstraints map[string]netmap.PlacementPolicy
func (c *Namespaces) UnmarshalJSON(data []byte) error {
namespaces := make(map[string]Namespace)
if err := json.Unmarshal(data, &namespaces); err != nil {
return err
}
for name, namespace := range namespaces {
namespace.Name = name
namespaces[name] = namespace
}
*c = namespaces
return nil
}
func (c *LocationConstraints) UnmarshalJSON(data []byte) error {
m := make(map[string]string)
if err := json.Unmarshal(data, &m); err != nil {
return err
}
*c = make(LocationConstraints, len(m))
for region, policy := range m {
var pp netmap.PlacementPolicy
if err := pp.DecodeString(policy); err == nil {
(*c)[region] = pp
continue
}
if err := pp.UnmarshalJSON([]byte(policy)); err == nil {
(*c)[region] = pp
continue
}
return fmt.Errorf("failed to parse location contraint '%s': '%s'", region, policy)
}
return nil
}
func (c LocationConstraints) MarshalJSON() ([]byte, error) {
m := make(map[string]string, len(c))
for region, policy := range c {
var sb strings.Builder
if err := policy.WriteStringTo(&sb); err != nil {
return nil, err
}
m[region] = sb.String()
}
return json.Marshal(m)
}

4
config/config.env
View file

@ -183,3 +183,7 @@ S3_GW_WEB_IDLE_TIMEOUT=30s
S3_GW_FROSTFSID_ENABLED=true
# FrostfsID contract hash (LE) or name in NNS.
S3_GW_FROSTFSID_CONTRACT=frostfsid.frostfs
# Namespaces configuration
S3_GW_NAMESPACES_CONFIG=namespaces.json

3
config/config.yaml
View file

@ -216,3 +216,6 @@ frostfsid:
enabled: true
# FrostfsID contract hash (LE) or name in NNS.
contract: frostfsid.frostfs
namespaces:
config: namespaces.json

39
docs/configuration.md
View file

@ -189,6 +189,7 @@ There are some custom types used for brevity:
| `features` | [Features configuration](#features-section) |
| `web` | [Web server configuration](#web-section) |
| `frostfsid` | [FrostfsID configuration](#frostfsid-section) |
| `namespaces` | [Namespaces configuration](#namespaces-section) |
### General section
@ -615,3 +616,41 @@ frostfsid:
|------------|----------|---------------|-------------------|----------------------------------------------------------------------------------------|
| `enabled` | `bool` | no | true | Enables check that allow requests only users that is registered in FrostfsID contract. |
| `contract` | `string` | no | frostfsid.frostfs | FrostfsID contract hash (LE) or name in NNS. |
# `namespaces` section
Namespaces configuration.
```yaml
namespaces:
config: namespace.json
```
| Parameter | Type | SIGHUP reload | Default value | Description |
|-----------|----------|---------------|---------------|-----------------------------------------------------|
| `config` | `string` | yes | | Path to json file with config value for namespaces. |
## `namespaces.config` subsection
Example of `namespaces.json`.
Note that config values from `namespaces.json` can override config values for default namespaces
(value for which are fetched from regular config value e.g. [placement-policy](#placement_policy-section)).
To override config values for default namespaces use namespace names that are provided in `kludge.default_namespaces`.
```json
{
"namespaces": {
"namespace1": {
"location_constraints": {
"default": "REP 3",
"test": "{\"replicas\":[{\"count\":1,\"selector\":\"\"}],\"containerBackupFactor\":0,\"selectors\":[],\"filters\":[],\"unique\":false}"
},
"copies_numbers": {
"default": [ 0 ],
"test": [ 1 ]
}
}
}
}
```

4
internal/logs/logs.go
View file

@ -40,6 +40,10 @@ const (
FailedToParseLocationConstraint = "failed to parse location constraint, it cannot be used" // Warn in cmd/s3-gw/app_settings.go
FailedToParseDefaultCopiesNumbers = "failed to parse 'default' copies numbers, default one will be used" // Warn in cmd/s3-gw/app_settings.go
FailedToParseCopiesNumbers = "failed to parse copies numbers, skip" // Warn in cmd/s3-gw/app_settings.go
DefaultNamespacesCannotBeEmpty = "default namespaces cannot be empty, defaults will be used" // Warn in cmd/s3-gw/app_settings.go
FailedToParseNamespacesConfig = "failed to unmarshal namespaces config" // Warn in cmd/s3-gw/app_settings.go
DefaultNamespaceConfigValuesBeOverwritten = "default namespace config value be overwritten by values from 'namespaces.config'" // Warn in cmd/s3-gw/app_settings.go
MultipleDefaultOverridesFound = "multiple default overrides found, only one will be used" // Warn in cmd/s3-gw/app_settings.go
FailedToParseDefaultDefaultLocationConstraint = "failed to parse default 'default' location constraint" // Fatal in cmd/s3-gw/app_settings.go
ConstraintAdded = "constraint added" // Info in ../../cmd/s3-gw/app_settings.go
SkipEmptyAddress = "skip, empty address" // Warn in ../../cmd/s3-gw/app_settings.go