From f663a1c125dbdbd35df65fe24a5996f725eb17d4 Mon Sep 17 00:00:00 2001 From: Leonard Lyubich Date: Tue, 1 Jun 2021 21:22:41 +0300 Subject: [PATCH] [#493] node/config: Implement sections of local object storage Signed-off-by: Leonard Lyubich --- cmd/neofs-node/config/engine/config.go | 34 +++++ .../shard/blobstor/blobovnicza/config.go | 93 ++++++++++++++ .../config/engine/shard/blobstor/config.go | 113 +++++++++++++++++ cmd/neofs-node/config/engine/shard/config.go | 60 +++++++++ .../config/engine/shard/gc/config.go | 59 +++++++++ .../config/engine/shard/metabase/config.go | 54 ++++++++ .../config/engine/shard/writecache/config.go | 117 ++++++++++++++++++ 7 files changed, 530 insertions(+) create mode 100644 cmd/neofs-node/config/engine/config.go create mode 100644 cmd/neofs-node/config/engine/shard/blobstor/blobovnicza/config.go create mode 100644 cmd/neofs-node/config/engine/shard/blobstor/config.go create mode 100644 cmd/neofs-node/config/engine/shard/config.go create mode 100644 cmd/neofs-node/config/engine/shard/gc/config.go create mode 100644 cmd/neofs-node/config/engine/shard/metabase/config.go create mode 100644 cmd/neofs-node/config/engine/shard/writecache/config.go diff --git a/cmd/neofs-node/config/engine/config.go b/cmd/neofs-node/config/engine/config.go new file mode 100644 index 000000000..67571d1ce --- /dev/null +++ b/cmd/neofs-node/config/engine/config.go @@ -0,0 +1,34 @@ +package engineconfig + +import ( + "strconv" + + "github.com/nspcc-dev/neofs-node/cmd/neofs-node/config" + shardconfig "github.com/nspcc-dev/neofs-node/cmd/neofs-node/config/engine/shard" +) + +// IterateShards iterates over subsections ["0":"N") (N - "shard_num" value) +// of "shard" subsection of "storage" section of c, wrap them into +// shardconfig.Config and passes to f. +// +// Panics if N is not a positive number. +func IterateShards(c *config.Config, f func(*shardconfig.Config)) { + c = c.Sub("storage") + + num := config.Uint(c, "shard_num") + if num == 0 { + panic("no shard configured") + } + + c = c.Sub("shard") + + for i := uint64(0); i < num; i++ { + si := strconv.FormatUint(i, 10) + + sc := shardconfig.From( + c.Sub(si), + ) + + f(sc) + } +} diff --git a/cmd/neofs-node/config/engine/shard/blobstor/blobovnicza/config.go b/cmd/neofs-node/config/engine/shard/blobstor/blobovnicza/config.go new file mode 100644 index 000000000..51ffa44dc --- /dev/null +++ b/cmd/neofs-node/config/engine/shard/blobstor/blobovnicza/config.go @@ -0,0 +1,93 @@ +package blobovniczaconfig + +import ( + "github.com/nspcc-dev/neofs-node/cmd/neofs-node/config" +) + +// Config is a wrapper over the config section +// which provides access to Blobovnicza configurations. +type Config config.Config + +// config defaults +const ( + // SizeDefault is a default limit of estimates of Blobovnicza size. + SizeDefault = 1 << 30 + + // ShallowDepthDefault is a default shallow dir depth. + ShallowDepthDefault = 2 + + // ShallowWidthDefault is a default shallow dir width. + ShallowWidthDefault = 16 + + // OpenedCacheSizeDefault is a default cache size of opened Blobovnicza's. + OpenedCacheSizeDefault = 16 +) + +// From wraps config section into Config. +func From(c *config.Config) *Config { + return (*Config)(c) +} + +// Size returns value of "size" config parameter. +// +// Returns SizeDefault if value is not a positive number. +func (x *Config) Size() uint64 { + s := config.UintSafe( + (*config.Config)(x), + "size", + ) + + if s > 0 { + return s + } + + return SizeDefault +} + +// ShallowDepth returns value of "shallow_depth" config parameter. +// +// Returns ShallowDepthDefault if value is not a positive number. +func (x *Config) ShallowDepth() uint64 { + d := config.UintSafe( + (*config.Config)(x), + "shallow_depth", + ) + + if d > 0 { + return d + } + + return ShallowDepthDefault +} + +// ShallowWidth returns value of "shallow_width" config parameter. +// +// Returns ShallowWidthDefault if value is not a positive number. +func (x *Config) ShallowWidth() uint64 { + d := config.UintSafe( + (*config.Config)(x), + "shallow_width", + ) + + if d > 0 { + return d + } + + return ShallowWidthDefault +} + +// OpenedCacheSize returns value of "opened_cache_size" config parameter. +// +// Returns OpenedCacheSizeDefault if value is not a positive number. +func (x *Config) OpenedCacheSize() int { + d := config.IntSafe( + (*config.Config)(x), + "opened_cache_size", + ) + + if d > 0 { + return int(d) + } + + return OpenedCacheSizeDefault +} diff --git a/cmd/neofs-node/config/engine/shard/blobstor/config.go b/cmd/neofs-node/config/engine/shard/blobstor/config.go new file mode 100644 index 000000000..419404637 --- /dev/null +++ b/cmd/neofs-node/config/engine/shard/blobstor/config.go @@ -0,0 +1,113 @@ +package blobstorconfig + +import ( + "os" + + "github.com/nspcc-dev/neofs-node/cmd/neofs-node/config" + blobovniczaconfig "github.com/nspcc-dev/neofs-node/cmd/neofs-node/config/engine/shard/blobstor/blobovnicza" + "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor/fstree" +) + +// Config is a wrapper over the config section +// which provides access to BlobStor configurations. +type Config config.Config + +// config defaults +const ( + // PermDefault are default permission bits for BlobStor data. + PermDefault = 0700 + + // ShallowDepthDefault is a default shallow dir depth. + ShallowDepthDefault = 4 + + // SmallSizeLimitDefault is a default limit of small objects payload in bytes. + SmallSizeLimitDefault = 1 << 20 +) + +// From wraps config section into Config. +func From(c *config.Config) *Config { + return (*Config)(c) +} + +// Path returns value of "path" config parameter. +// +// Panics if value is not a non-empty string. +func (x *Config) Path() string { + p := config.String( + (*config.Config)(x), + "path", + ) + + if p == "" { + panic("blobstor path not set") + } + + return p +} + +// Perm returns value of "perm" config parameter as a os.FileMode. +// +// Returns PermDefault if value is not a non-zero number. +func (x *Config) Perm() os.FileMode { + p := config.UintSafe( + (*config.Config)(x), + "perm", + ) + + if p == 0 { + p = PermDefault + } + + return os.FileMode(p) +} + +// ShallowDepth returns value of "shallow_depth" config parameter. +// +// Returns ShallowDepthDefault if value is out of +// [1:fstree.MaxDepth] range. +func (x *Config) ShallowDepth() int { + d := config.IntSafe( + (*config.Config)(x), + "shallow_depth", + ) + + if d >= 1 && d <= fstree.MaxDepth { + return int(d) + } + + return ShallowDepthDefault +} + +// Compress returns value of "compress" config parameter. +// +// Returns false if value is not a valid bool. +func (x *Config) Compress() bool { + return config.BoolSafe( + (*config.Config)(x), + "compress", + ) +} + +// SmallSizeLimit returns value of "small_size_limit" config parameter. +// +// Returns SmallSizeLimitDefault if value is not a positive number. +func (x *Config) SmallSizeLimit() uint64 { + l := config.UintSafe( + (*config.Config)(x), + "small_size_limit", + ) + + if l > 0 { + return l + } + + return SmallSizeLimitDefault +} + +// Blobovnicza returns "blobovnicza" subsection as a blobovniczaconfig.Config. +func (x *Config) Blobovnicza() *blobovniczaconfig.Config { + return blobovniczaconfig.From( + (*config.Config)(x). + Sub("blobovnicza"), + ) +} diff --git a/cmd/neofs-node/config/engine/shard/config.go b/cmd/neofs-node/config/engine/shard/config.go new file mode 100644 index 000000000..d34f1760e --- /dev/null +++ b/cmd/neofs-node/config/engine/shard/config.go @@ -0,0 +1,60 @@ +package shardconfig + +import ( + "github.com/nspcc-dev/neofs-node/cmd/neofs-node/config" + blobstorconfig "github.com/nspcc-dev/neofs-node/cmd/neofs-node/config/engine/shard/blobstor" + gcconfig "github.com/nspcc-dev/neofs-node/cmd/neofs-node/config/engine/shard/gc" + metabaseconfig "github.com/nspcc-dev/neofs-node/cmd/neofs-node/config/engine/shard/metabase" + writecacheconfig "github.com/nspcc-dev/neofs-node/cmd/neofs-node/config/engine/shard/writecache" +) + +// Config is a wrapper over the config section +// which provides access to Shard configurations. +type Config config.Config + +// From wraps config section into Config. +func From(c *config.Config) *Config { + return (*Config)(c) +} + +// UseWriteCache returns value of "use_write_cache" config parameter. +// +// Panics if value is not a valid bool. +func (x *Config) UseWriteCache() bool { + return config.Bool( + (*config.Config)(x), + "use_write_cache", + ) +} + +// BlobStor returns "blobstor" subsection as a blobstorconfig.Config. +func (x *Config) BlobStor() *blobstorconfig.Config { + return blobstorconfig.From( + (*config.Config)(x). + Sub("blobstor"), + ) +} + +// Metabase returns "metabase" subsection as a metabaseconfig.Config. +func (x *Config) Metabase() *metabaseconfig.Config { + return metabaseconfig.From( + (*config.Config)(x). + Sub("metabase"), + ) +} + +// WriteCache returns "writecache" subsection as a writecacheconfig.Config. +func (x *Config) WriteCache() *writecacheconfig.Config { + return writecacheconfig.From( + (*config.Config)(x). + Sub("writecache"), + ) +} + +// GC returns "gc" subsection as a gcconfig.Config. +func (x *Config) GC() *gcconfig.Config { + return gcconfig.From( + (*config.Config)(x). + Sub("gc"), + ) +} diff --git a/cmd/neofs-node/config/engine/shard/gc/config.go b/cmd/neofs-node/config/engine/shard/gc/config.go new file mode 100644 index 000000000..3ff87ec6b --- /dev/null +++ b/cmd/neofs-node/config/engine/shard/gc/config.go @@ -0,0 +1,59 @@ +package gcconfig + +import ( + "time" + + "github.com/nspcc-dev/neofs-node/cmd/neofs-node/config" +) + +// Config is a wrapper over the config section +// which provides access to Shard's GC configurations. +type Config config.Config + +// config defaults +const ( + // RemoverBatchSizeDefault is a default batch size for Shard GC's remover. + RemoverBatchSizeDefault = 100 + + // RemoverSleepInterval is a default sleep interval of Shard GC's remover. + RemoverSleepIntervalDefault = time.Minute +) + +// From wraps config section into Config. +func From(c *config.Config) *Config { + return (*Config)(c) +} + +// RemoverBatchSize returns value of "remover_batch_size" +// config parameter. +// +// Returns RemoverBatchSizeDefault if value is not a positive number. +func (x *Config) RemoverBatchSize() int { + s := config.IntSafe( + (*config.Config)(x), + "remover_batch_size", + ) + + if s > 0 { + return int(s) + } + + return RemoverBatchSizeDefault +} + +// RemoverSleepInterval returns value of "remover_sleep_interval" +// config parameter. +// +// Returns RemoverSleepIntervalDefault if value is not a positive number. +func (x *Config) RemoverSleepInterval() time.Duration { + s := config.DurationSafe( + (*config.Config)(x), + "remover_sleep_interval", + ) + + if s > 0 { + return s + } + + return RemoverSleepIntervalDefault +} diff --git a/cmd/neofs-node/config/engine/shard/metabase/config.go b/cmd/neofs-node/config/engine/shard/metabase/config.go new file mode 100644 index 000000000..a5df71e6c --- /dev/null +++ b/cmd/neofs-node/config/engine/shard/metabase/config.go @@ -0,0 +1,54 @@ +package metabaseconfig + +import ( + "os" + + "github.com/nspcc-dev/neofs-node/cmd/neofs-node/config" +) + +// Config is a wrapper over the config section +// which provides access to Metabase configurations. +type Config config.Config + +// config defaults +const ( + // PermDefault is a default permission bits for metabase file. + PermDefault = 0700 +) + +// From wraps config section into Config. +func From(c *config.Config) *Config { + return (*Config)(c) +} + +// Path returns value of "path" config parameter. +// +// Panics if value is not a non-empty string. +func (x *Config) Path() string { + p := config.String( + (*config.Config)(x), + "path", + ) + + if p == "" { + panic("metabase path not set") + } + + return p +} + +// Perm returns value of "perm" config parameter as a os.FileMode. +// +// Returns PermDefault if value is not a positive number. +func (x *Config) Perm() os.FileMode { + p := config.UintSafe( + (*config.Config)(x), + "perm", + ) + + if p == 0 { + p = PermDefault + } + + return os.FileMode(p) +} diff --git a/cmd/neofs-node/config/engine/shard/writecache/config.go b/cmd/neofs-node/config/engine/shard/writecache/config.go new file mode 100644 index 000000000..a08080a95 --- /dev/null +++ b/cmd/neofs-node/config/engine/shard/writecache/config.go @@ -0,0 +1,117 @@ +package writecacheconfig + +import ( + "github.com/nspcc-dev/neofs-node/cmd/neofs-node/config" +) + +// Config is a wrapper over the config section +// which provides access to WriteCache configurations. +type Config config.Config + +// config defaults +const ( + // MemSizeDefault is a default memory size. + MemSizeDefault = 1 << 30 + + // SmallSizeDefault is a default size of small objects. + SmallSizeDefault = 32 << 10 + + // MaxSizeDefault is a default value of the object payload size limit. + MaxSizeDefault = 64 << 20 + + // WorkersNumberDefault is a default number of workers. + WorkersNumberDefault = 20 +) + +// From wraps config section into Config. +func From(c *config.Config) *Config { + return (*Config)(c) +} + +// Path returns value of "path" config parameter. +// +// Panics if value is not a non-empty string. +func (x *Config) Path() string { + p := config.String( + (*config.Config)(x), + "path", + ) + + if p == "" { + panic("write cache path not set") + } + + return p +} + +// MemSize returns value of "mem_size" config parameter. +// +// Returns MemSizeDefault if value is not a positive number. +func (x *Config) MemSize() uint64 { + s := config.UintSafe( + (*config.Config)(x), + "mem_size", + ) + + if s > 0 { + return s + } + + return MemSizeDefault +} + +// MaxDBSize returns value of "db_size" config parameter. +func (x *Config) MaxDBSize() uint64 { + return config.UintSafe( + (*config.Config)(x), + "db_size", + ) +} + +// SmallObjectSize returns value of "small_size" config parameter. +// +// Returns SmallSizeDefault if value is not a positive number. +func (x *Config) SmallObjectSize() uint64 { + s := config.UintSafe( + (*config.Config)(x), + "small_size", + ) + + if s > 0 { + return s + } + + return SmallSizeDefault +} + +// MaxObjectSize returns value of "max_size" config parameter. +// +// Returns MaxSizeDefault if value is not a positive number. +func (x *Config) MaxObjectSize() uint64 { + s := config.UintSafe( + (*config.Config)(x), + "max_size", + ) + + if s > 0 { + return s + } + + return MaxSizeDefault +} + +// WorkersNumber returns value of "workers_number" config parameter. +// +// Returns WorkersNumberDefault if value is not a positive number. +func (x *Config) WorkersNumber() int { + c := config.IntSafe( + (*config.Config)(x), + "workers_number", + ) + + if c > 0 { + return int(c) + } + + return WorkersNumberDefault +}