[#493] node/config: Implement sections of local object storage

Signed-off-by: Leonard Lyubich <leonard@nspcc.ru>
This commit is contained in:
Leonard Lyubich 2021-06-01 21:22:41 +03:00 committed by Leonard Lyubich
parent e26dc0a6e3
commit f663a1c125
7 changed files with 530 additions and 0 deletions

View file

@ -0,0 +1,34 @@
package engineconfig
import (
"strconv"
"github.com/nspcc-dev/neofs-node/cmd/neofs-node/config"
shardconfig "github.com/nspcc-dev/neofs-node/cmd/neofs-node/config/engine/shard"
)
// IterateShards iterates over subsections ["0":"N") (N - "shard_num" value)
// of "shard" subsection of "storage" section of c, wrap them into
// shardconfig.Config and passes to f.
//
// Panics if N is not a positive number.
func IterateShards(c *config.Config, f func(*shardconfig.Config)) {
c = c.Sub("storage")
num := config.Uint(c, "shard_num")
if num == 0 {
panic("no shard configured")
}
c = c.Sub("shard")
for i := uint64(0); i < num; i++ {
si := strconv.FormatUint(i, 10)
sc := shardconfig.From(
c.Sub(si),
)
f(sc)
}
}

View file

@ -0,0 +1,93 @@
package blobovniczaconfig
import (
"github.com/nspcc-dev/neofs-node/cmd/neofs-node/config"
)
// Config is a wrapper over the config section
// which provides access to Blobovnicza configurations.
type Config config.Config
// config defaults
const (
// SizeDefault is a default limit of estimates of Blobovnicza size.
SizeDefault = 1 << 30
// ShallowDepthDefault is a default shallow dir depth.
ShallowDepthDefault = 2
// ShallowWidthDefault is a default shallow dir width.
ShallowWidthDefault = 16
// OpenedCacheSizeDefault is a default cache size of opened Blobovnicza's.
OpenedCacheSizeDefault = 16
)
// From wraps config section into Config.
func From(c *config.Config) *Config {
return (*Config)(c)
}
// Size returns value of "size" config parameter.
//
// Returns SizeDefault if value is not a positive number.
func (x *Config) Size() uint64 {
s := config.UintSafe(
(*config.Config)(x),
"size",
)
if s > 0 {
return s
}
return SizeDefault
}
// ShallowDepth returns value of "shallow_depth" config parameter.
//
// Returns ShallowDepthDefault if value is not a positive number.
func (x *Config) ShallowDepth() uint64 {
d := config.UintSafe(
(*config.Config)(x),
"shallow_depth",
)
if d > 0 {
return d
}
return ShallowDepthDefault
}
// ShallowWidth returns value of "shallow_width" config parameter.
//
// Returns ShallowWidthDefault if value is not a positive number.
func (x *Config) ShallowWidth() uint64 {
d := config.UintSafe(
(*config.Config)(x),
"shallow_width",
)
if d > 0 {
return d
}
return ShallowWidthDefault
}
// OpenedCacheSize returns value of "opened_cache_size" config parameter.
//
// Returns OpenedCacheSizeDefault if value is not a positive number.
func (x *Config) OpenedCacheSize() int {
d := config.IntSafe(
(*config.Config)(x),
"opened_cache_size",
)
if d > 0 {
return int(d)
}
return OpenedCacheSizeDefault
}

View file

@ -0,0 +1,113 @@
package blobstorconfig
import (
"os"
"github.com/nspcc-dev/neofs-node/cmd/neofs-node/config"
blobovniczaconfig "github.com/nspcc-dev/neofs-node/cmd/neofs-node/config/engine/shard/blobstor/blobovnicza"
"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor/fstree"
)
// Config is a wrapper over the config section
// which provides access to BlobStor configurations.
type Config config.Config
// config defaults
const (
// PermDefault are default permission bits for BlobStor data.
PermDefault = 0700
// ShallowDepthDefault is a default shallow dir depth.
ShallowDepthDefault = 4
// SmallSizeLimitDefault is a default limit of small objects payload in bytes.
SmallSizeLimitDefault = 1 << 20
)
// From wraps config section into Config.
func From(c *config.Config) *Config {
return (*Config)(c)
}
// Path returns value of "path" config parameter.
//
// Panics if value is not a non-empty string.
func (x *Config) Path() string {
p := config.String(
(*config.Config)(x),
"path",
)
if p == "" {
panic("blobstor path not set")
}
return p
}
// Perm returns value of "perm" config parameter as a os.FileMode.
//
// Returns PermDefault if value is not a non-zero number.
func (x *Config) Perm() os.FileMode {
p := config.UintSafe(
(*config.Config)(x),
"perm",
)
if p == 0 {
p = PermDefault
}
return os.FileMode(p)
}
// ShallowDepth returns value of "shallow_depth" config parameter.
//
// Returns ShallowDepthDefault if value is out of
// [1:fstree.MaxDepth] range.
func (x *Config) ShallowDepth() int {
d := config.IntSafe(
(*config.Config)(x),
"shallow_depth",
)
if d >= 1 && d <= fstree.MaxDepth {
return int(d)
}
return ShallowDepthDefault
}
// Compress returns value of "compress" config parameter.
//
// Returns false if value is not a valid bool.
func (x *Config) Compress() bool {
return config.BoolSafe(
(*config.Config)(x),
"compress",
)
}
// SmallSizeLimit returns value of "small_size_limit" config parameter.
//
// Returns SmallSizeLimitDefault if value is not a positive number.
func (x *Config) SmallSizeLimit() uint64 {
l := config.UintSafe(
(*config.Config)(x),
"small_size_limit",
)
if l > 0 {
return l
}
return SmallSizeLimitDefault
}
// Blobovnicza returns "blobovnicza" subsection as a blobovniczaconfig.Config.
func (x *Config) Blobovnicza() *blobovniczaconfig.Config {
return blobovniczaconfig.From(
(*config.Config)(x).
Sub("blobovnicza"),
)
}

View file

@ -0,0 +1,60 @@
package shardconfig
import (
"github.com/nspcc-dev/neofs-node/cmd/neofs-node/config"
blobstorconfig "github.com/nspcc-dev/neofs-node/cmd/neofs-node/config/engine/shard/blobstor"
gcconfig "github.com/nspcc-dev/neofs-node/cmd/neofs-node/config/engine/shard/gc"
metabaseconfig "github.com/nspcc-dev/neofs-node/cmd/neofs-node/config/engine/shard/metabase"
writecacheconfig "github.com/nspcc-dev/neofs-node/cmd/neofs-node/config/engine/shard/writecache"
)
// Config is a wrapper over the config section
// which provides access to Shard configurations.
type Config config.Config
// From wraps config section into Config.
func From(c *config.Config) *Config {
return (*Config)(c)
}
// UseWriteCache returns value of "use_write_cache" config parameter.
//
// Panics if value is not a valid bool.
func (x *Config) UseWriteCache() bool {
return config.Bool(
(*config.Config)(x),
"use_write_cache",
)
}
// BlobStor returns "blobstor" subsection as a blobstorconfig.Config.
func (x *Config) BlobStor() *blobstorconfig.Config {
return blobstorconfig.From(
(*config.Config)(x).
Sub("blobstor"),
)
}
// Metabase returns "metabase" subsection as a metabaseconfig.Config.
func (x *Config) Metabase() *metabaseconfig.Config {
return metabaseconfig.From(
(*config.Config)(x).
Sub("metabase"),
)
}
// WriteCache returns "writecache" subsection as a writecacheconfig.Config.
func (x *Config) WriteCache() *writecacheconfig.Config {
return writecacheconfig.From(
(*config.Config)(x).
Sub("writecache"),
)
}
// GC returns "gc" subsection as a gcconfig.Config.
func (x *Config) GC() *gcconfig.Config {
return gcconfig.From(
(*config.Config)(x).
Sub("gc"),
)
}

View file

@ -0,0 +1,59 @@
package gcconfig
import (
"time"
"github.com/nspcc-dev/neofs-node/cmd/neofs-node/config"
)
// Config is a wrapper over the config section
// which provides access to Shard's GC configurations.
type Config config.Config
// config defaults
const (
// RemoverBatchSizeDefault is a default batch size for Shard GC's remover.
RemoverBatchSizeDefault = 100
// RemoverSleepInterval is a default sleep interval of Shard GC's remover.
RemoverSleepIntervalDefault = time.Minute
)
// From wraps config section into Config.
func From(c *config.Config) *Config {
return (*Config)(c)
}
// RemoverBatchSize returns value of "remover_batch_size"
// config parameter.
//
// Returns RemoverBatchSizeDefault if value is not a positive number.
func (x *Config) RemoverBatchSize() int {
s := config.IntSafe(
(*config.Config)(x),
"remover_batch_size",
)
if s > 0 {
return int(s)
}
return RemoverBatchSizeDefault
}
// RemoverSleepInterval returns value of "remover_sleep_interval"
// config parameter.
//
// Returns RemoverSleepIntervalDefault if value is not a positive number.
func (x *Config) RemoverSleepInterval() time.Duration {
s := config.DurationSafe(
(*config.Config)(x),
"remover_sleep_interval",
)
if s > 0 {
return s
}
return RemoverSleepIntervalDefault
}

View file

@ -0,0 +1,54 @@
package metabaseconfig
import (
"os"
"github.com/nspcc-dev/neofs-node/cmd/neofs-node/config"
)
// Config is a wrapper over the config section
// which provides access to Metabase configurations.
type Config config.Config
// config defaults
const (
// PermDefault is a default permission bits for metabase file.
PermDefault = 0700
)
// From wraps config section into Config.
func From(c *config.Config) *Config {
return (*Config)(c)
}
// Path returns value of "path" config parameter.
//
// Panics if value is not a non-empty string.
func (x *Config) Path() string {
p := config.String(
(*config.Config)(x),
"path",
)
if p == "" {
panic("metabase path not set")
}
return p
}
// Perm returns value of "perm" config parameter as a os.FileMode.
//
// Returns PermDefault if value is not a positive number.
func (x *Config) Perm() os.FileMode {
p := config.UintSafe(
(*config.Config)(x),
"perm",
)
if p == 0 {
p = PermDefault
}
return os.FileMode(p)
}

View file

@ -0,0 +1,117 @@
package writecacheconfig
import (
"github.com/nspcc-dev/neofs-node/cmd/neofs-node/config"
)
// Config is a wrapper over the config section
// which provides access to WriteCache configurations.
type Config config.Config
// config defaults
const (
// MemSizeDefault is a default memory size.
MemSizeDefault = 1 << 30
// SmallSizeDefault is a default size of small objects.
SmallSizeDefault = 32 << 10
// MaxSizeDefault is a default value of the object payload size limit.
MaxSizeDefault = 64 << 20
// WorkersNumberDefault is a default number of workers.
WorkersNumberDefault = 20
)
// From wraps config section into Config.
func From(c *config.Config) *Config {
return (*Config)(c)
}
// Path returns value of "path" config parameter.
//
// Panics if value is not a non-empty string.
func (x *Config) Path() string {
p := config.String(
(*config.Config)(x),
"path",
)
if p == "" {
panic("write cache path not set")
}
return p
}
// MemSize returns value of "mem_size" config parameter.
//
// Returns MemSizeDefault if value is not a positive number.
func (x *Config) MemSize() uint64 {
s := config.UintSafe(
(*config.Config)(x),
"mem_size",
)
if s > 0 {
return s
}
return MemSizeDefault
}
// MaxDBSize returns value of "db_size" config parameter.
func (x *Config) MaxDBSize() uint64 {
return config.UintSafe(
(*config.Config)(x),
"db_size",
)
}
// SmallObjectSize returns value of "small_size" config parameter.
//
// Returns SmallSizeDefault if value is not a positive number.
func (x *Config) SmallObjectSize() uint64 {
s := config.UintSafe(
(*config.Config)(x),
"small_size",
)
if s > 0 {
return s
}
return SmallSizeDefault
}
// MaxObjectSize returns value of "max_size" config parameter.
//
// Returns MaxSizeDefault if value is not a positive number.
func (x *Config) MaxObjectSize() uint64 {
s := config.UintSafe(
(*config.Config)(x),
"max_size",
)
if s > 0 {
return s
}
return MaxSizeDefault
}
// WorkersNumber returns value of "workers_number" config parameter.
//
// Returns WorkersNumberDefault if value is not a positive number.
func (x *Config) WorkersNumber() int {
c := config.IntSafe(
(*config.Config)(x),
"workers_number",
)
if c > 0 {
return int(c)
}
return WorkersNumberDefault
}