From f663a1c125dbdbd35df65fe24a5996f725eb17d4 Mon Sep 17 00:00:00 2001
From: Leonard Lyubich <leonard@nspcc.ru>
Date: Tue, 1 Jun 2021 21:22:41 +0300
Subject: [PATCH] [#493] node/config: Implement sections of local object
 storage

Signed-off-by: Leonard Lyubich <leonard@nspcc.ru>
---
 cmd/neofs-node/config/engine/config.go        |  34 +++++
 .../shard/blobstor/blobovnicza/config.go      |  93 ++++++++++++++
 .../config/engine/shard/blobstor/config.go    | 113 +++++++++++++++++
 cmd/neofs-node/config/engine/shard/config.go  |  60 +++++++++
 .../config/engine/shard/gc/config.go          |  59 +++++++++
 .../config/engine/shard/metabase/config.go    |  54 ++++++++
 .../config/engine/shard/writecache/config.go  | 117 ++++++++++++++++++
 7 files changed, 530 insertions(+)
 create mode 100644 cmd/neofs-node/config/engine/config.go
 create mode 100644 cmd/neofs-node/config/engine/shard/blobstor/blobovnicza/config.go
 create mode 100644 cmd/neofs-node/config/engine/shard/blobstor/config.go
 create mode 100644 cmd/neofs-node/config/engine/shard/config.go
 create mode 100644 cmd/neofs-node/config/engine/shard/gc/config.go
 create mode 100644 cmd/neofs-node/config/engine/shard/metabase/config.go
 create mode 100644 cmd/neofs-node/config/engine/shard/writecache/config.go

diff --git a/cmd/neofs-node/config/engine/config.go b/cmd/neofs-node/config/engine/config.go
new file mode 100644
index 00000000..67571d1c
--- /dev/null
+++ b/cmd/neofs-node/config/engine/config.go
@@ -0,0 +1,34 @@
+package engineconfig
+
+import (
+	"strconv"
+
+	"github.com/nspcc-dev/neofs-node/cmd/neofs-node/config"
+	shardconfig "github.com/nspcc-dev/neofs-node/cmd/neofs-node/config/engine/shard"
+)
+
+// IterateShards iterates over subsections ["0":"N") (N - "shard_num" value)
+// of "shard" subsection of "storage" section of c, wrap them into
+// shardconfig.Config and passes to f.
+//
+// Panics if N is not a positive number.
+func IterateShards(c *config.Config, f func(*shardconfig.Config)) {
+	c = c.Sub("storage")
+
+	num := config.Uint(c, "shard_num")
+	if num == 0 {
+		panic("no shard configured")
+	}
+
+	c = c.Sub("shard")
+
+	for i := uint64(0); i < num; i++ {
+		si := strconv.FormatUint(i, 10)
+
+		sc := shardconfig.From(
+			c.Sub(si),
+		)
+
+		f(sc)
+	}
+}
diff --git a/cmd/neofs-node/config/engine/shard/blobstor/blobovnicza/config.go b/cmd/neofs-node/config/engine/shard/blobstor/blobovnicza/config.go
new file mode 100644
index 00000000..51ffa44d
--- /dev/null
+++ b/cmd/neofs-node/config/engine/shard/blobstor/blobovnicza/config.go
@@ -0,0 +1,93 @@
+package blobovniczaconfig
+
+import (
+	"github.com/nspcc-dev/neofs-node/cmd/neofs-node/config"
+)
+
+// Config is a wrapper over the config section
+// which provides access to Blobovnicza configurations.
+type Config config.Config
+
+// config defaults
+const (
+	// SizeDefault is a default limit of estimates of Blobovnicza size.
+	SizeDefault = 1 << 30
+
+	// ShallowDepthDefault is a default shallow dir depth.
+	ShallowDepthDefault = 2
+
+	// ShallowWidthDefault is a default shallow dir width.
+	ShallowWidthDefault = 16
+
+	// OpenedCacheSizeDefault is a default cache size of opened Blobovnicza's.
+	OpenedCacheSizeDefault = 16
+)
+
+// From wraps config section into Config.
+func From(c *config.Config) *Config {
+	return (*Config)(c)
+}
+
+// Size returns value of "size" config parameter.
+//
+// Returns SizeDefault if value is not a positive number.
+func (x *Config) Size() uint64 {
+	s := config.UintSafe(
+		(*config.Config)(x),
+		"size",
+	)
+
+	if s > 0 {
+		return s
+	}
+
+	return SizeDefault
+}
+
+// ShallowDepth returns value of "shallow_depth" config parameter.
+//
+// Returns ShallowDepthDefault if value is not a positive number.
+func (x *Config) ShallowDepth() uint64 {
+	d := config.UintSafe(
+		(*config.Config)(x),
+		"shallow_depth",
+	)
+
+	if d > 0 {
+		return d
+	}
+
+	return ShallowDepthDefault
+}
+
+// ShallowWidth returns value of "shallow_width" config parameter.
+//
+// Returns ShallowWidthDefault if value is not a positive number.
+func (x *Config) ShallowWidth() uint64 {
+	d := config.UintSafe(
+		(*config.Config)(x),
+		"shallow_width",
+	)
+
+	if d > 0 {
+		return d
+	}
+
+	return ShallowWidthDefault
+}
+
+// OpenedCacheSize returns value of "opened_cache_size" config parameter.
+//
+// Returns OpenedCacheSizeDefault if value is not a positive number.
+func (x *Config) OpenedCacheSize() int {
+	d := config.IntSafe(
+		(*config.Config)(x),
+		"opened_cache_size",
+	)
+
+	if d > 0 {
+		return int(d)
+	}
+
+	return OpenedCacheSizeDefault
+}
diff --git a/cmd/neofs-node/config/engine/shard/blobstor/config.go b/cmd/neofs-node/config/engine/shard/blobstor/config.go
new file mode 100644
index 00000000..41940463
--- /dev/null
+++ b/cmd/neofs-node/config/engine/shard/blobstor/config.go
@@ -0,0 +1,113 @@
+package blobstorconfig
+
+import (
+	"os"
+
+	"github.com/nspcc-dev/neofs-node/cmd/neofs-node/config"
+	blobovniczaconfig "github.com/nspcc-dev/neofs-node/cmd/neofs-node/config/engine/shard/blobstor/blobovnicza"
+	"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor/fstree"
+)
+
+// Config is a wrapper over the config section
+// which provides access to BlobStor configurations.
+type Config config.Config
+
+// config defaults
+const (
+	// PermDefault are default permission bits for BlobStor data.
+	PermDefault = 0700
+
+	// ShallowDepthDefault is a default shallow dir depth.
+	ShallowDepthDefault = 4
+
+	// SmallSizeLimitDefault is a default limit of small objects payload in bytes.
+	SmallSizeLimitDefault = 1 << 20
+)
+
+// From wraps config section into Config.
+func From(c *config.Config) *Config {
+	return (*Config)(c)
+}
+
+// Path returns value of "path" config parameter.
+//
+// Panics if value is not a non-empty string.
+func (x *Config) Path() string {
+	p := config.String(
+		(*config.Config)(x),
+		"path",
+	)
+
+	if p == "" {
+		panic("blobstor path not set")
+	}
+
+	return p
+}
+
+// Perm returns value of "perm" config parameter as a os.FileMode.
+//
+// Returns PermDefault if value is not a non-zero number.
+func (x *Config) Perm() os.FileMode {
+	p := config.UintSafe(
+		(*config.Config)(x),
+		"perm",
+	)
+
+	if p == 0 {
+		p = PermDefault
+	}
+
+	return os.FileMode(p)
+}
+
+// ShallowDepth returns value of "shallow_depth" config parameter.
+//
+// Returns ShallowDepthDefault if value is out of
+// [1:fstree.MaxDepth] range.
+func (x *Config) ShallowDepth() int {
+	d := config.IntSafe(
+		(*config.Config)(x),
+		"shallow_depth",
+	)
+
+	if d >= 1 && d <= fstree.MaxDepth {
+		return int(d)
+	}
+
+	return ShallowDepthDefault
+}
+
+// Compress returns value of "compress" config parameter.
+//
+// Returns false if value is not a valid bool.
+func (x *Config) Compress() bool {
+	return config.BoolSafe(
+		(*config.Config)(x),
+		"compress",
+	)
+}
+
+// SmallSizeLimit returns value of "small_size_limit" config parameter.
+//
+// Returns SmallSizeLimitDefault if value is not a positive number.
+func (x *Config) SmallSizeLimit() uint64 {
+	l := config.UintSafe(
+		(*config.Config)(x),
+		"small_size_limit",
+	)
+
+	if l > 0 {
+		return l
+	}
+
+	return SmallSizeLimitDefault
+}
+
+// Blobovnicza returns "blobovnicza" subsection as a blobovniczaconfig.Config.
+func (x *Config) Blobovnicza() *blobovniczaconfig.Config {
+	return blobovniczaconfig.From(
+		(*config.Config)(x).
+			Sub("blobovnicza"),
+	)
+}
diff --git a/cmd/neofs-node/config/engine/shard/config.go b/cmd/neofs-node/config/engine/shard/config.go
new file mode 100644
index 00000000..d34f1760
--- /dev/null
+++ b/cmd/neofs-node/config/engine/shard/config.go
@@ -0,0 +1,60 @@
+package shardconfig
+
+import (
+	"github.com/nspcc-dev/neofs-node/cmd/neofs-node/config"
+	blobstorconfig "github.com/nspcc-dev/neofs-node/cmd/neofs-node/config/engine/shard/blobstor"
+	gcconfig "github.com/nspcc-dev/neofs-node/cmd/neofs-node/config/engine/shard/gc"
+	metabaseconfig "github.com/nspcc-dev/neofs-node/cmd/neofs-node/config/engine/shard/metabase"
+	writecacheconfig "github.com/nspcc-dev/neofs-node/cmd/neofs-node/config/engine/shard/writecache"
+)
+
+// Config is a wrapper over the config section
+// which provides access to Shard configurations.
+type Config config.Config
+
+// From wraps config section into Config.
+func From(c *config.Config) *Config {
+	return (*Config)(c)
+}
+
+// UseWriteCache returns value of "use_write_cache" config parameter.
+//
+// Panics if value is not a valid bool.
+func (x *Config) UseWriteCache() bool {
+	return config.Bool(
+		(*config.Config)(x),
+		"use_write_cache",
+	)
+}
+
+// BlobStor returns "blobstor" subsection as a blobstorconfig.Config.
+func (x *Config) BlobStor() *blobstorconfig.Config {
+	return blobstorconfig.From(
+		(*config.Config)(x).
+			Sub("blobstor"),
+	)
+}
+
+// Metabase returns "metabase" subsection as a metabaseconfig.Config.
+func (x *Config) Metabase() *metabaseconfig.Config {
+	return metabaseconfig.From(
+		(*config.Config)(x).
+			Sub("metabase"),
+	)
+}
+
+// WriteCache returns "writecache" subsection as a writecacheconfig.Config.
+func (x *Config) WriteCache() *writecacheconfig.Config {
+	return writecacheconfig.From(
+		(*config.Config)(x).
+			Sub("writecache"),
+	)
+}
+
+// GC returns "gc" subsection as a gcconfig.Config.
+func (x *Config) GC() *gcconfig.Config {
+	return gcconfig.From(
+		(*config.Config)(x).
+			Sub("gc"),
+	)
+}
diff --git a/cmd/neofs-node/config/engine/shard/gc/config.go b/cmd/neofs-node/config/engine/shard/gc/config.go
new file mode 100644
index 00000000..3ff87ec6
--- /dev/null
+++ b/cmd/neofs-node/config/engine/shard/gc/config.go
@@ -0,0 +1,59 @@
+package gcconfig
+
+import (
+	"time"
+
+	"github.com/nspcc-dev/neofs-node/cmd/neofs-node/config"
+)
+
+// Config is a wrapper over the config section
+// which provides access to Shard's GC configurations.
+type Config config.Config
+
+// config defaults
+const (
+	// RemoverBatchSizeDefault is a default batch size for Shard GC's remover.
+	RemoverBatchSizeDefault = 100
+
+	// RemoverSleepInterval is a default sleep interval of Shard GC's remover.
+	RemoverSleepIntervalDefault = time.Minute
+)
+
+// From wraps config section into Config.
+func From(c *config.Config) *Config {
+	return (*Config)(c)
+}
+
+// RemoverBatchSize returns value of "remover_batch_size"
+// config parameter.
+//
+// Returns RemoverBatchSizeDefault if value is not a positive number.
+func (x *Config) RemoverBatchSize() int {
+	s := config.IntSafe(
+		(*config.Config)(x),
+		"remover_batch_size",
+	)
+
+	if s > 0 {
+		return int(s)
+	}
+
+	return RemoverBatchSizeDefault
+}
+
+// RemoverSleepInterval returns value of "remover_sleep_interval"
+// config parameter.
+//
+// Returns RemoverSleepIntervalDefault if value is not a positive number.
+func (x *Config) RemoverSleepInterval() time.Duration {
+	s := config.DurationSafe(
+		(*config.Config)(x),
+		"remover_sleep_interval",
+	)
+
+	if s > 0 {
+		return s
+	}
+
+	return RemoverSleepIntervalDefault
+}
diff --git a/cmd/neofs-node/config/engine/shard/metabase/config.go b/cmd/neofs-node/config/engine/shard/metabase/config.go
new file mode 100644
index 00000000..a5df71e6
--- /dev/null
+++ b/cmd/neofs-node/config/engine/shard/metabase/config.go
@@ -0,0 +1,54 @@
+package metabaseconfig
+
+import (
+	"os"
+
+	"github.com/nspcc-dev/neofs-node/cmd/neofs-node/config"
+)
+
+// Config is a wrapper over the config section
+// which provides access to Metabase configurations.
+type Config config.Config
+
+// config defaults
+const (
+	// PermDefault is a default permission bits for metabase file.
+	PermDefault = 0700
+)
+
+// From wraps config section into Config.
+func From(c *config.Config) *Config {
+	return (*Config)(c)
+}
+
+// Path returns value of "path" config parameter.
+//
+// Panics if value is not a non-empty string.
+func (x *Config) Path() string {
+	p := config.String(
+		(*config.Config)(x),
+		"path",
+	)
+
+	if p == "" {
+		panic("metabase path not set")
+	}
+
+	return p
+}
+
+// Perm returns value of "perm" config parameter as a os.FileMode.
+//
+// Returns PermDefault if value is not a positive number.
+func (x *Config) Perm() os.FileMode {
+	p := config.UintSafe(
+		(*config.Config)(x),
+		"perm",
+	)
+
+	if p == 0 {
+		p = PermDefault
+	}
+
+	return os.FileMode(p)
+}
diff --git a/cmd/neofs-node/config/engine/shard/writecache/config.go b/cmd/neofs-node/config/engine/shard/writecache/config.go
new file mode 100644
index 00000000..a08080a9
--- /dev/null
+++ b/cmd/neofs-node/config/engine/shard/writecache/config.go
@@ -0,0 +1,117 @@
+package writecacheconfig
+
+import (
+	"github.com/nspcc-dev/neofs-node/cmd/neofs-node/config"
+)
+
+// Config is a wrapper over the config section
+// which provides access to WriteCache configurations.
+type Config config.Config
+
+// config defaults
+const (
+	// MemSizeDefault is a default memory size.
+	MemSizeDefault = 1 << 30
+
+	// SmallSizeDefault is a default size of small objects.
+	SmallSizeDefault = 32 << 10
+
+	// MaxSizeDefault is a default value of the object payload size limit.
+	MaxSizeDefault = 64 << 20
+
+	// WorkersNumberDefault is a default number of workers.
+	WorkersNumberDefault = 20
+)
+
+// From wraps config section into Config.
+func From(c *config.Config) *Config {
+	return (*Config)(c)
+}
+
+// Path returns value of "path" config parameter.
+//
+// Panics if value is not a non-empty string.
+func (x *Config) Path() string {
+	p := config.String(
+		(*config.Config)(x),
+		"path",
+	)
+
+	if p == "" {
+		panic("write cache path not set")
+	}
+
+	return p
+}
+
+// MemSize returns value of "mem_size" config parameter.
+//
+// Returns MemSizeDefault if value is not a positive number.
+func (x *Config) MemSize() uint64 {
+	s := config.UintSafe(
+		(*config.Config)(x),
+		"mem_size",
+	)
+
+	if s > 0 {
+		return s
+	}
+
+	return MemSizeDefault
+}
+
+// MaxDBSize returns value of "db_size" config parameter.
+func (x *Config) MaxDBSize() uint64 {
+	return config.UintSafe(
+		(*config.Config)(x),
+		"db_size",
+	)
+}
+
+// SmallObjectSize returns value of "small_size" config parameter.
+//
+// Returns SmallSizeDefault if value is not a positive number.
+func (x *Config) SmallObjectSize() uint64 {
+	s := config.UintSafe(
+		(*config.Config)(x),
+		"small_size",
+	)
+
+	if s > 0 {
+		return s
+	}
+
+	return SmallSizeDefault
+}
+
+// MaxObjectSize returns value of "max_size" config parameter.
+//
+// Returns MaxSizeDefault if value is not a positive number.
+func (x *Config) MaxObjectSize() uint64 {
+	s := config.UintSafe(
+		(*config.Config)(x),
+		"max_size",
+	)
+
+	if s > 0 {
+		return s
+	}
+
+	return MaxSizeDefault
+}
+
+// WorkersNumber returns value of "workers_number" config parameter.
+//
+// Returns WorkersNumberDefault if value is not a positive number.
+func (x *Config) WorkersNumber() int {
+	c := config.IntSafe(
+		(*config.Config)(x),
+		"workers_number",
+	)
+
+	if c > 0 {
+		return int(c)
+	}
+
+	return WorkersNumberDefault
+}