TrueCloudLab/frostfs-node
20
1
Fork 27
Code Issues 101 Pull requests 6 Projects Releases 1 Activity Actions

[#1523] neofs-node: Refactor configuration

Browse source 
1. Move compression parameters to the `shard` section.
2. Allow to use multiple sub-storage components in the blobstor.

Signed-off-by: Evgenii Stratonikov <evgeniy@nspcc.ru>
This commit is contained in:
Evgenii Stratonikov 2022-07-11 15:34:17 +03:00 committed by fyrchik
parent 13cdbde2e2
commit 26b4a258e0
37 changed files with 595 additions and 419 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

59
cmd/neofs-node/config.go
View file

@ -15,6 +15,8 @@ import (
contractsconfig "github.com/nspcc-dev/neofs-node/cmd/neofs-node/config/contracts"
engineconfig "github.com/nspcc-dev/neofs-node/cmd/neofs-node/config/engine"
shardconfig "github.com/nspcc-dev/neofs-node/cmd/neofs-node/config/engine/shard"
blobovniczaconfig "github.com/nspcc-dev/neofs-node/cmd/neofs-node/config/engine/shard/blobstor/blobovnicza"
fstreeconfig "github.com/nspcc-dev/neofs-node/cmd/neofs-node/config/engine/shard/blobstor/fstree"
loggerconfig "github.com/nspcc-dev/neofs-node/cmd/neofs-node/config/logger"
metricsconfig "github.com/nspcc-dev/neofs-node/cmd/neofs-node/config/metrics"
nodeconfig "github.com/nspcc-dev/neofs-node/cmd/neofs-node/config/node"
@ -22,6 +24,8 @@ import (
"github.com/nspcc-dev/neofs-node/pkg/core/container"
netmapCore "github.com/nspcc-dev/neofs-node/pkg/core/netmap"
"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor"
"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor/blobovniczatree"
"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor/fstree"
"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/engine"
meta "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/metabase"
"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/pilorama"
@ -47,6 +51,7 @@ import (
"github.com/nspcc-dev/neofs-node/pkg/util/logger"
"github.com/nspcc-dev/neofs-node/pkg/util/state"
"github.com/nspcc-dev/neofs-sdk-go/netmap"
objectSDK "github.com/nspcc-dev/neofs-sdk-go/object"
"github.com/nspcc-dev/neofs-sdk-go/user"
"github.com/nspcc-dev/neofs-sdk-go/version"
"github.com/panjf2000/ants/v2"
@ -418,7 +423,7 @@ func initShardOptions(c *cfg) {
}
blobStorCfg := sc.BlobStor()
blobovniczaCfg := blobStorCfg.Blobovnicza()
storages := blobStorCfg.Storages()
metabaseCfg := sc.Metabase()
gcCfg := sc.GC()
@ -426,19 +431,47 @@ func initShardOptions(c *cfg) {
piloramaCfg := sc.Pilorama()
if config.BoolSafe(c.appCfg.Sub("tree"), "enabled") {
piloramaPath := piloramaCfg.Path()
if piloramaPath == "" {
piloramaPath = filepath.Join(blobStorCfg.Path(), "pilorama.db")
}
piloramaOpts = []pilorama.Option{
pilorama.WithPath(piloramaPath),
pilorama.WithPath(piloramaCfg.Path()),
pilorama.WithPerm(piloramaCfg.Perm()),
pilorama.WithNoSync(piloramaCfg.NoSync()),
pilorama.WithMaxBatchSize(piloramaCfg.MaxBatchSize()),
pilorama.WithMaxBatchDelay(piloramaCfg.MaxBatchDelay())}
}
var st []blobstor.SubStorage
for i := range storages {
switch storages[i].Type() {
case "blobovniczas":
sub := blobovniczaconfig.From((*config.Config)(storages[i]))
lim := sc.SmallSizeLimit()
st = append(st, blobstor.SubStorage{
Storage: blobovniczatree.NewBlobovniczaTree(
blobovniczatree.WithLogger(c.log),
blobovniczatree.WithRootPath(storages[i].Path()),
blobovniczatree.WithPermissions(storages[i].Perm()),
blobovniczatree.WithBlobovniczaSize(sub.Size()),
blobovniczatree.WithBlobovniczaShallowDepth(sub.ShallowDepth()),
blobovniczatree.WithBlobovniczaShallowWidth(sub.ShallowWidth()),
blobovniczatree.WithOpenedCacheSize(sub.OpenedCacheSize())),
Policy: func(_ *objectSDK.Object, data []byte) bool {
return uint64(len(data)) < lim
},
})
case "fstree":
sub := fstreeconfig.From((*config.Config)(storages[i]))
st = append(st, blobstor.SubStorage{
Storage: fstree.New(
fstree.WithPath(storages[i].Path()),
fstree.WithPerm(storages[i].Perm()),
fstree.WithDepth(sub.Depth())),
Policy: func(_ *objectSDK.Object, data []byte) bool {
return true
},
})
}
}
metaPath := metabaseCfg.Path()
metaPerm := metabaseCfg.BoltDB().Perm()
fatalOnErr(util.MkdirAllX(filepath.Dir(metaPath), metaPerm))
@ -453,15 +486,9 @@ func initShardOptions(c *cfg) {
shard.WithRefillMetabase(sc.RefillMetabase()),
shard.WithMode(sc.Mode()),
shard.WithBlobStorOptions(
blobstor.WithRootPath(blobStorCfg.Path()),
blobstor.WithCompressObjects(blobStorCfg.Compress()),
blobstor.WithRootPerm(blobStorCfg.Perm()),
blobstor.WithShallowDepth(blobStorCfg.ShallowDepth()),
blobstor.WithSmallSizeLimit(blobStorCfg.SmallSizeLimit()),
blobstor.WithBlobovniczaSize(blobovniczaCfg.Size()),
blobstor.WithBlobovniczaShallowDepth(blobovniczaCfg.ShallowDepth()),
blobstor.WithBlobovniczaShallowWidth(blobovniczaCfg.ShallowWidth()),
blobstor.WithBlobovniczaOpenedCacheSize(blobovniczaCfg.OpenedCacheSize()),
blobstor.WithCompressObjects(sc.Compress()),
blobstor.WithUncompressableContentTypes(sc.UncompressableContentTypes()),
blobstor.WithStorages(st),
blobstor.WithLogger(c.log),
),
shard.WithMetaBaseOptions(

2
cmd/neofs-node/config/engine/config.go
View file

@ -39,7 +39,7 @@ func IterateShards(c *config.Config, required bool, f func(*shardconfig.Config))
// must have different paths, so if it is missing, the shard is not here.
// At the same time checking for "blobstor" section doesn't work proper
// with configuration via the environment.
if (*config.Config)(sc).Value("blobstor.path") == nil {
if (*config.Config)(sc).Value("metabase.path") == nil {
break
}
(*config.Config)(sc).SetDefault(def)

38
cmd/neofs-node/config/engine/config_test.go
View file

@ -8,6 +8,8 @@ import (
"github.com/nspcc-dev/neofs-node/cmd/neofs-node/config"
engineconfig "github.com/nspcc-dev/neofs-node/cmd/neofs-node/config/engine"
shardconfig "github.com/nspcc-dev/neofs-node/cmd/neofs-node/config/engine/shard"
blobovniczaconfig "github.com/nspcc-dev/neofs-node/cmd/neofs-node/config/engine/shard/blobstor/blobovnicza"
fstreeconfig "github.com/nspcc-dev/neofs-node/cmd/neofs-node/config/engine/shard/blobstor/fstree"
piloramaconfig "github.com/nspcc-dev/neofs-node/cmd/neofs-node/config/engine/shard/pilorama"
configtest "github.com/nspcc-dev/neofs-node/cmd/neofs-node/config/test"
"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/shard/mode"
@ -53,7 +55,7 @@ func TestEngineSection(t *testing.T) {
wc := sc.WriteCache()
meta := sc.Metabase()
blob := sc.BlobStor()
blz := blob.Blobovnicza()
ss := blob.Storages()
pl := sc.Pilorama()
gc := sc.GC()
@ -79,18 +81,23 @@ func TestEngineSection(t *testing.T) {
require.Equal(t, 100, meta.BoltDB().MaxBatchSize())
require.Equal(t, 10*time.Millisecond, meta.BoltDB().MaxBatchDelay())
require.Equal(t, "tmp/0/blob", blob.Path())
require.EqualValues(t, 0644, blob.Perm())
require.Equal(t, true, blob.Compress())
require.Equal(t, []string{"audio/*", "video/*"}, blob.UncompressableContentTypes())
require.EqualValues(t, 5, blob.ShallowDepth())
require.EqualValues(t, 102400, blob.SmallSizeLimit())
require.Equal(t, true, sc.Compress())
require.Equal(t, []string{"audio/*", "video/*"}, sc.UncompressableContentTypes())
require.EqualValues(t, 102400, sc.SmallSizeLimit())
require.Equal(t, 2, len(ss))
blz := blobovniczaconfig.From((*config.Config)(ss[0]))
require.Equal(t, "tmp/0/blob/blobovnicza", ss[0].Path())
require.EqualValues(t, 0644, blz.BoltDB().Perm())
require.EqualValues(t, 4194304, blz.Size())
require.EqualValues(t, 1, blz.ShallowDepth())
require.EqualValues(t, 4, blz.ShallowWidth())
require.EqualValues(t, 50, blz.OpenedCacheSize())
require.Equal(t, "tmp/0/blob", ss[1].Path())
require.EqualValues(t, 0644, ss[1].Perm())
require.EqualValues(t, 5, fstreeconfig.From((*config.Config)(ss[1])).Depth())
require.EqualValues(t, 150, gc.RemoverBatchSize())
require.Equal(t, 2*time.Minute, gc.RemoverSleepInterval())
@ -117,18 +124,23 @@ func TestEngineSection(t *testing.T) {
require.Equal(t, 200, meta.BoltDB().MaxBatchSize())
require.Equal(t, 20*time.Millisecond, meta.BoltDB().MaxBatchDelay())
require.Equal(t, "tmp/1/blob", blob.Path())
require.EqualValues(t, 0644, blob.Perm())
require.Equal(t, false, blob.Compress())
require.Equal(t, []string(nil), blob.UncompressableContentTypes())
require.EqualValues(t, 5, blob.ShallowDepth())
require.EqualValues(t, 102400, blob.SmallSizeLimit())
require.Equal(t, false, sc.Compress())
require.Equal(t, []string(nil), sc.UncompressableContentTypes())
require.EqualValues(t, 102400, sc.SmallSizeLimit())
require.Equal(t, 2, len(ss))
blz := blobovniczaconfig.From((*config.Config)(ss[0]))
require.Equal(t, "tmp/1/blob/blobovnicza", ss[0].Path())
require.EqualValues(t, 4194304, blz.Size())
require.EqualValues(t, 1, blz.ShallowDepth())
require.EqualValues(t, 4, blz.ShallowWidth())
require.EqualValues(t, 50, blz.OpenedCacheSize())
require.Equal(t, "tmp/1/blob", ss[1].Path())
require.EqualValues(t, 0644, ss[1].Perm())
require.EqualValues(t, 5, fstreeconfig.From((*config.Config)(ss[1])).Depth())
require.EqualValues(t, 200, gc.RemoverBatchSize())
require.Equal(t, 5*time.Minute, gc.RemoverSleepInterval())

5
cmd/neofs-node/config/engine/shard/blobstor/blobovnicza/config.go
View file

@ -29,6 +29,11 @@ func From(c *config.Config) *Config {
return (*Config)(c)
}
// Type returns the storage type.
func (x *Config) Type() string {
return "blobovnicza"
}
// Size returns the value of "size" config parameter.
//
// Returns SizeDefault if the value is not a positive number.

122
cmd/neofs-node/config/engine/shard/blobstor/config.go
View file

@ -1,122 +1,36 @@
package blobstorconfig
import (
"io/fs"
"strconv"
"github.com/nspcc-dev/neofs-node/cmd/neofs-node/config"
blobovniczaconfig "github.com/nspcc-dev/neofs-node/cmd/neofs-node/config/engine/shard/blobstor/blobovnicza"
"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor/fstree"
"github.com/nspcc-dev/neofs-node/cmd/neofs-node/config/engine/shard/blobstor/storage"
)
// Config is a wrapper over the config section
// which provides access to BlobStor configurations.
type Config config.Config
// config defaults
const (
// PermDefault are default permission bits for BlobStor data.
PermDefault = 0660
// ShallowDepthDefault is a default shallow dir depth.
ShallowDepthDefault = 4
// SmallSizeLimitDefault is a default limit of small objects payload in bytes.
SmallSizeLimitDefault = 1 << 20
)
// From wraps config section into Config.
func From(c *config.Config) *Config {
return (*Config)(c)
}
// Path returns the value of "path" config parameter.
//
// Panics if the value is not a non-empty string.
func (x *Config) Path() string {
p := config.String(
(*config.Config)(x),
"path",
)
if p == "" {
panic("blobstor path not set")
// Storages returns the value of storage subcomponents.
func (x *Config) Storages() []*storage.Config {
var ss []*storage.Config
for i := 0; ; i++ {
typ := config.String(
(*config.Config)(x),
strconv.Itoa(i)+".type")
switch typ {
case "":
return ss
case "fstree", "blobovnicza":
sub := storage.From((*config.Config)(x).Sub(strconv.Itoa(i)))
ss = append(ss, sub)
default:
panic("invalid type")
}
}
return p
}
// Perm returns the value of "perm" config parameter as a fs.FileMode.
//
// Returns PermDefault if the value is not a non-zero number.
func (x *Config) Perm() fs.FileMode {
p := config.UintSafe(
(*config.Config)(x),
"perm",
)
if p == 0 {
p = PermDefault
}
return fs.FileMode(p)
}
// ShallowDepth returns the value of "depth" config parameter.
//
// Returns ShallowDepthDefault if the value is out of
// [1:fstree.MaxDepth] range.
func (x *Config) ShallowDepth() int {
d := config.IntSafe(
(*config.Config)(x),
"depth",
)
if d >= 1 && d <= fstree.MaxDepth {
return int(d)
}
return ShallowDepthDefault
}
// Compress returns the value of "compress" config parameter.
//
// Returns false if the value is not a valid bool.
func (x *Config) Compress() bool {
return config.BoolSafe(
(*config.Config)(x),
"compress",
)
}
// UncompressableContentTypes returns the value of "compress_skip_content_types" config parameter.
//
// Returns nil if a the value is missing or is invalid.
func (x *Config) UncompressableContentTypes() []string {
return config.StringSliceSafe(
(*config.Config)(x),
"compression_exclude_content_types")
}
// SmallSizeLimit returns the value of "small_object_size" config parameter.
//
// Returns SmallSizeLimitDefault if the value is not a positive number.
func (x *Config) SmallSizeLimit() uint64 {
l := config.SizeInBytesSafe(
(*config.Config)(x),
"small_object_size",
)
if l > 0 {
return l
}
return SmallSizeLimitDefault
}
// Blobovnicza returns "blobovnicza" subsection as a blobovniczaconfig.Config.
func (x *Config) Blobovnicza() *blobovniczaconfig.Config {
return blobovniczaconfig.From(
(*config.Config)(x).
Sub("blobovnicza"),
)
}

40
cmd/neofs-node/config/engine/shard/blobstor/fstree/config.go Normal file
View file

@ -0,0 +1,40 @@
package fstree
import (
"github.com/nspcc-dev/neofs-node/cmd/neofs-node/config"
"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor/fstree"
)
// Config is a wrapper over the config section
// which provides access to Blobovnicza configurations.
type Config config.Config
// DepthDefault is a default shallow dir depth.
const DepthDefault = 4
// From wraps config section into Config.
func From(c *config.Config) *Config {
return (*Config)(c)
}
// Type returns the storage type.
func (x *Config) Type() string {
return "fstree"
}
// Depth returns the value of "depth" config parameter.
//
// Returns DepthDefault if the value is out of
// [1:fstree.MaxDepth] range.
func (x *Config) Depth() int {
d := config.IntSafe(
(*config.Config)(x),
"depth",
)
if d >= 1 && d <= fstree.MaxDepth {
return int(d)
}
return DepthDefault
}

55
cmd/neofs-node/config/engine/shard/blobstor/storage/config.go Normal file
View file

@ -0,0 +1,55 @@
package storage
import (
"io/fs"
"github.com/nspcc-dev/neofs-node/cmd/neofs-node/config"
)
type Config config.Config
// PermDefault are default permission bits for BlobStor data.
const PermDefault = 0660
func From(x *config.Config) *Config {
return (*Config)(x)
}
// Type returns storage type.
func (x *Config) Type() string {
return config.String(
(*config.Config)(x),
"type")
}
// Path returns the value of "path" config parameter.
//
// Panics if the value is not a non-empty string.
func (x *Config) Path() string {
p := config.String(
(*config.Config)(x),
"path",
)
if p == "" {
panic("blobstor path not set")
}
return p
}
// Perm returns the value of "perm" config parameter as a fs.FileMode.
//
// Returns PermDefault if the value is not a non-zero number.
func (x *Config) Perm() fs.FileMode {
p := config.UintSafe(
(*config.Config)(x),
"perm",
)
if p == 0 {
p = PermDefault
}
return fs.FileMode(p)
}

38
cmd/neofs-node/config/engine/shard/config.go
View file

@ -16,11 +16,49 @@ import (
// which provides access to Shard configurations.
type Config config.Config
// SmallSizeLimitDefault is a default limit of small objects payload in bytes.
const SmallSizeLimitDefault = 1 << 20
// From wraps config section into Config.
func From(c *config.Config) *Config {
return (*Config)(c)
}
// Compress returns the value of "compress" config parameter.
//
// Returns false if the value is not a valid bool.
func (x *Config) Compress() bool {
return config.BoolSafe(
(*config.Config)(x),
"compress",
)
}
// UncompressableContentTypes returns the value of "compress_skip_content_types" config parameter.
//
// Returns nil if a the value is missing or is invalid.
func (x *Config) UncompressableContentTypes() []string {
return config.StringSliceSafe(
(*config.Config)(x),
"compression_exclude_content_types")
}
// SmallSizeLimit returns the value of "small_object_size" config parameter.
//
// Returns SmallSizeLimitDefault if the value is not a positive number.
func (x *Config) SmallSizeLimit() uint64 {
l := config.SizeInBytesSafe(
(*config.Config)(x),
"small_object_size",
)
if l > 0 {
return l
}
return SmallSizeLimitDefault
}
// BlobStor returns "blobstor" subsection as a blobstorconfig.Config.
func (x *Config) BlobStor() *blobstorconfig.Config {
return blobstorconfig.From(

47
config/example/node.env
View file

@ -101,17 +101,22 @@ NEOFS_STORAGE_SHARD_0_METABASE_PERM=0644
NEOFS_STORAGE_SHARD_0_METABASE_MAX_BATCH_SIZE=100
NEOFS_STORAGE_SHARD_0_METABASE_MAX_BATCH_DELAY=10ms
### Blobstor config
NEOFS_STORAGE_SHARD_0_BLOBSTOR_PATH=tmp/0/blob
NEOFS_STORAGE_SHARD_0_BLOBSTOR_PERM=0644
NEOFS_STORAGE_SHARD_0_BLOBSTOR_COMPRESS=true
NEOFS_STORAGE_SHARD_0_BLOBSTOR_COMPRESSION_EXCLUDE_CONTENT_TYPES="audio/* video/*"
NEOFS_STORAGE_SHARD_0_BLOBSTOR_DEPTH=5
NEOFS_STORAGE_SHARD_0_BLOBSTOR_SMALL_OBJECT_SIZE=102400
NEOFS_STORAGE_SHARD_0_COMPRESS=true
NEOFS_STORAGE_SHARD_0_COMPRESSION_EXCLUDE_CONTENT_TYPES="audio/* video/*"
NEOFS_STORAGE_SHARD_0_SMALL_OBJECT_SIZE=102400
### Blobovnicza config
NEOFS_STORAGE_SHARD_0_BLOBSTOR_BLOBOVNICZA_SIZE=4194304
NEOFS_STORAGE_SHARD_0_BLOBSTOR_BLOBOVNICZA_DEPTH=1
NEOFS_STORAGE_SHARD_0_BLOBSTOR_BLOBOVNICZA_WIDTH=4
NEOFS_STORAGE_SHARD_0_BLOBSTOR_BLOBOVNICZA_OPENED_CACHE_CAPACITY=50
NEOFS_STORAGE_SHARD_0_BLOBSTOR_0_PATH=tmp/0/blob/blobovnicza
NEOFS_STORAGE_SHARD_0_BLOBSTOR_0_PERM=0644
NEOFS_STORAGE_SHARD_0_BLOBSTOR_0_TYPE=blobovnicza
NEOFS_STORAGE_SHARD_0_BLOBSTOR_0_SIZE=4194304
NEOFS_STORAGE_SHARD_0_BLOBSTOR_0_DEPTH=1
NEOFS_STORAGE_SHARD_0_BLOBSTOR_0_WIDTH=4
NEOFS_STORAGE_SHARD_0_BLOBSTOR_0_OPENED_CACHE_CAPACITY=50
### FSTree config
NEOFS_STORAGE_SHARD_0_BLOBSTOR_1_TYPE=fstree
NEOFS_STORAGE_SHARD_0_BLOBSTOR_1_PATH=tmp/0/blob
NEOFS_STORAGE_SHARD_0_BLOBSTOR_1_PERM=0644
NEOFS_STORAGE_SHARD_0_BLOBSTOR_1_DEPTH=5
### Pilorama config
NEOFS_STORAGE_SHARD_0_PILORAMA_PATH="tmp/0/blob/pilorama.db"
NEOFS_STORAGE_SHARD_0_PILORAMA_MAX_BATCH_DELAY=10ms
@ -141,16 +146,20 @@ NEOFS_STORAGE_SHARD_1_METABASE_PERM=0644
NEOFS_STORAGE_SHARD_1_METABASE_MAX_BATCH_SIZE=200
NEOFS_STORAGE_SHARD_1_METABASE_MAX_BATCH_DELAY=20ms
### Blobstor config
NEOFS_STORAGE_SHARD_1_BLOBSTOR_PATH=tmp/1/blob
NEOFS_STORAGE_SHARD_1_BLOBSTOR_PERM=0644
NEOFS_STORAGE_SHARD_1_BLOBSTOR_COMPRESS=false
NEOFS_STORAGE_SHARD_1_BLOBSTOR_DEPTH=5
NEOFS_STORAGE_SHARD_1_BLOBSTOR_SMALL_OBJECT_SIZE=102400
NEOFS_STORAGE_SHARD_1_COMPRESS=false
NEOFS_STORAGE_SHARD_1_SMALL_OBJECT_SIZE=102400
### Blobovnicza config
NEOFS_STORAGE_SHARD_1_BLOBSTOR_BLOBOVNICZA_SIZE=4194304
NEOFS_STORAGE_SHARD_1_BLOBSTOR_BLOBOVNICZA_DEPTH=1
NEOFS_STORAGE_SHARD_1_BLOBSTOR_BLOBOVNICZA_WIDTH=4
NEOFS_STORAGE_SHARD_1_BLOBSTOR_BLOBOVNICZA_OPENED_CACHE_CAPACITY=50
NEOFS_STORAGE_SHARD_1_BLOBSTOR_0_TYPE=blobovnicza
NEOFS_STORAGE_SHARD_1_BLOBSTOR_0_PATH=tmp/1/blob/blobovnicza
NEOFS_STORAGE_SHARD_1_BLOBSTOR_0_SIZE=4194304
NEOFS_STORAGE_SHARD_1_BLOBSTOR_0_DEPTH=1
NEOFS_STORAGE_SHARD_1_BLOBSTOR_0_WIDTH=4
NEOFS_STORAGE_SHARD_1_BLOBSTOR_0_OPENED_CACHE_CAPACITY=50
### FSTree config
NEOFS_STORAGE_SHARD_1_BLOBSTOR_1_TYPE=fstree
NEOFS_STORAGE_SHARD_1_BLOBSTOR_1_PATH=tmp/1/blob
NEOFS_STORAGE_SHARD_1_BLOBSTOR_1_PERM=0644
NEOFS_STORAGE_SHARD_1_BLOBSTOR_1_DEPTH=5
### Pilorama config
NEOFS_STORAGE_SHARD_1_PILORAMA_PATH="tmp/1/blob/pilorama.db"
NEOFS_STORAGE_SHARD_1_PILORAMA_PERM=0644

50
config/example/node.json
View file

@ -147,22 +147,28 @@
"max_batch_size": 100,
"max_batch_delay": "10ms"
},
"blobstor": {
"path": "tmp/0/blob",
"perm": "0644",
"compress": true,
"compression_exclude_content_types": [
"audio/*", "video/*"
],
"depth": 5,
"small_object_size": 102400,
"blobovnicza": {
"compress": true,
"compression_exclude_content_types": [
"audio/*", "video/*"
],
"small_object_size": 102400,
"blobstor": [
{
"type": "blobovnicza",
"path": "tmp/0/blob/blobovnicza",
"perm": "0644",
"size": 4194304,
"depth": 1,
"width": 4,
"opened_cache_capacity": 50
},
{
"type": "fstree",
"path": "tmp/0/blob",
"perm": "0644",
"depth": 5
}
},
],
"pilorama": {
"path": "tmp/0/blob/pilorama.db",
"max_batch_delay": "10ms",
@ -191,19 +197,25 @@
"max_batch_size": 200,
"max_batch_delay": "20ms"
},
"blobstor": {
"path": "tmp/1/blob",
"perm": "0644",
"compress": false,
"depth": 5,
"small_object_size": 102400,
"blobovnicza": {
"compress": false,
"small_object_size": 102400,
"blobstor": [
{
"type": "blobovnicza",
"path": "tmp/1/blob/blobovnicza",
"perm": "0644",
"size": 4194304,
"depth": 1,
"width": 4,
"opened_cache_capacity": 50
},
{
"type": "fstree",
"path": "tmp/1/blob",
"perm": "0644",
"depth": 5
}
},
],
"pilorama": {
"path": "tmp/1/blob/pilorama.db",
"perm": "0644",

34
config/example/node.yaml
View file

@ -126,17 +126,17 @@ storage:
max_batch_delay: 5ms # maximum delay for a batch of operations to be executed
max_batch_size: 100 # maximum amount of operations in a single batch
blobstor:
compress: false # turn on/off zstd(level 3) compression of stored objects
perm: 0644 # permissions for blobstor files(directories: +x for current user and group)
depth: 5 # max depth of object tree storage in FS
small_object_size: 100 kb # size threshold for "small" objects which are cached in key-value DB, not in FS, bytes
compress: false # turn on/off zstd(level 3) compression of stored objects
small_object_size: 100 kb # size threshold for "small" objects which are cached in key-value DB, not in FS, bytes
blobovnicza:
size: 4m # approximate size limit of single blobovnicza instance, total size will be: size*width^(depth+1), bytes
blobstor:
- size: 4m # approximate size limit of single blobovnicza instance, total size will be: size*width^(depth+1), bytes
perm: 0644 # permissions for blobstor files(directories: +x for current user and group)
depth: 1 # max depth of object tree storage in key-value DB
width: 4 # max width of object tree storage in key-value DB
opened_cache_capacity: 50 # maximum number of opened database files
- perm: 0644 # permissions for blobstor files(directories: +x for current user and group)
depth: 5 # max depth of object tree storage in FS
gc:
remover_batch_size: 200 # number of objects to be removed by the garbage collector
@ -156,12 +156,16 @@ storage:
max_batch_size: 100
max_batch_delay: 10ms
compress: true # turn on/off zstd(level 3) compression of stored objects
compression_exclude_content_types:
- audio/*
- video/*
blobstor:
path: tmp/0/blob # blobstor path
compress: true # turn on/off zstd(level 3) compression of stored objects
compression_exclude_content_types:
- audio/*
- video/*
- type: blobovnicza
path: tmp/0/blob/blobovnicza
- type: fstree
path: tmp/0/blob # blobstor path
pilorama:
path: tmp/0/blob/pilorama.db # path to the pilorama database. If omitted, `pilorama.db` file is created blobstor.path
@ -181,8 +185,10 @@ storage:
path: tmp/1/meta # metabase path
blobstor:
path: tmp/1/blob # blobstor path
- type: blobovnicza
path: tmp/1/blob/blobovnicza
- type: fstree
path: tmp/1/blob # blobstor path
pilorama:
path: tmp/1/blob/pilorama.db

8
pkg/local_object_storage/blobstor/blobovniczatree/blobovnicza.go
View file

@ -11,6 +11,7 @@ import (
"github.com/nspcc-dev/hrw"
"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobovnicza"
"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor/common"
"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor/compression"
apistatus "github.com/nspcc-dev/neofs-sdk-go/client/status"
objectSDK "github.com/nspcc-dev/neofs-sdk-go/object"
oid "github.com/nspcc-dev/neofs-sdk-go/object/id"
@ -76,8 +77,6 @@ type Blobovniczas struct {
// list of active (opened, non-filled) Blobovniczas
activeMtx sync.RWMutex
active map[string]blobovniczaWithIndex
onClose []func()
}
type blobovniczaWithIndex struct {
@ -887,3 +886,8 @@ func u64FromHexString(str string) uint64 {
func (b *Blobovniczas) Type() string {
return "blobovniczas"
}
// SetCompressor implements common.Storage.
func (b *Blobovniczas) SetCompressor(cc *compression.CConfig) {
b.CConfig = cc
}

14
pkg/local_object_storage/blobstor/blobovniczatree/control.go
View file

@ -19,16 +19,6 @@ func (b *Blobovniczas) Open(readOnly bool) error {
func (b *Blobovniczas) Init() error {
b.log.Debug("initializing Blobovnicza's")
err := b.CConfig.Init()
if err != nil {
return err
}
b.onClose = append(b.onClose, func() {
if err := b.CConfig.Close(); err != nil {
b.log.Debug("can't close zstd compressor", zap.String("err", err.Error()))
}
})
if b.readOnly {
b.log.Debug("read-only mode, skip blobovniczas initialization...")
return nil
@ -78,9 +68,5 @@ func (b *Blobovniczas) Close() error {
b.activeMtx.Unlock()
for i := range b.onClose {
b.onClose[i]()
}
return nil
}

7
pkg/local_object_storage/blobstor/blobovniczatree/option.go
View file

@ -36,7 +36,6 @@ func initConfig(c *cfg) {
openedCacheSize: defaultOpenedCacheSize,
blzShallowDepth: defaultBlzShallowDepth,
blzShallowWidth: defaultBlzShallowWidth,
CConfig: new(compression.CConfig),
}
}
@ -53,12 +52,6 @@ func WithPermissions(perm fs.FileMode) Option {
}
}
func WithCompressionConfig(cc *compression.CConfig) Option {
return func(c *cfg) {
c.CConfig = cc
}
}
func WithBlobovniczaShallowWidth(width uint64) Option {
return func(c *cfg) {
c.blzShallowWidth = width

139
pkg/local_object_storage/blobstor/blobstor.go
View file

@ -1,12 +1,8 @@
package blobstor
import (
"encoding/hex"
"io/fs"
"path/filepath"
"sync"
"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor/blobovniczatree"
"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor/common"
"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor/compression"
"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor/fstree"
@ -28,8 +24,6 @@ type BlobStor struct {
modeMtx sync.RWMutex
mode mode.Mode
storage [2]SubStorage
}
type Info = fstree.Info
@ -39,37 +33,12 @@ type Option func(*cfg)
type cfg struct {
compression.CConfig
fsTreeDepth int
fsTreeInfo fstree.Info
smallSizeLimit uint64
log *logger.Logger
blzOpts []blobovniczatree.Option
log *logger.Logger
storage []SubStorage
}
const (
defaultShallowDepth = 4
defaultPerm = 0700
defaultSmallSizeLimit = 1 << 20 // 1MB
)
const blobovniczaDir = "blobovnicza"
func initConfig(c *cfg) {
*c = cfg{
fsTreeDepth: defaultShallowDepth,
fsTreeInfo: Info{
Permissions: defaultPerm,
RootPath: "./",
},
smallSizeLimit: defaultSmallSizeLimit,
log: zap.L(),
}
c.blzOpts = []blobovniczatree.Option{blobovniczatree.WithCompressionConfig(&c.CConfig)}
c.log = zap.L()
}
// New creates, initializes and returns new BlobStor instance.
@ -81,23 +50,10 @@ func New(opts ...Option) *BlobStor {
opts[i](&bs.cfg)
}
bs.storage[0].Storage = blobovniczatree.NewBlobovniczaTree(bs.blzOpts...)
bs.storage[0].Policy = func(_ *objectSDK.Object, data []byte) bool {
return uint64(len(data)) <= bs.cfg.smallSizeLimit
for i := range bs.storage {
bs.storage[i].Storage.SetCompressor(&bs.CConfig)
}
bs.storage[1].Storage = &fstree.FSTree{
Info: bs.cfg.fsTreeInfo,
Depth: bs.cfg.fsTreeDepth,
DirNameLen: hex.EncodedLen(fstree.DirNameLen),
CConfig: &bs.cfg.CConfig,
}
bs.storage[1].Policy = func(*objectSDK.Object, []byte) bool {
return true
}
bs.blzOpts = nil
return bs
}
@ -106,17 +62,17 @@ func (b *BlobStor) SetLogger(l *zap.Logger) {
b.log = l
}
// WithShallowDepth returns option to set the
// depth of the object file subdirectory tree.
//
// Depth is reduced to maximum value in case of overflow.
func WithShallowDepth(depth int) Option {
// WithStorages provides sub-blobstors.
func WithStorages(st []SubStorage) Option {
return func(c *cfg) {
if depth > fstree.MaxDepth {
depth = fstree.MaxDepth
}
c.storage = st
}
}
c.fsTreeDepth = depth
// WithLogger returns option to specify BlobStor's logger.
func WithLogger(l *logger.Logger) Option {
return func(c *cfg) {
c.log = l.With(zap.String("component", "BlobStor"))
}
}
@ -141,70 +97,3 @@ func WithUncompressableContentTypes(values []string) Option {
c.UncompressableContentTypes = values
}
}
// WithRootPath returns option to set path to root directory
// of the fs tree to write the objects.
func WithRootPath(rootDir string) Option {
return func(c *cfg) {
c.fsTreeInfo.RootPath = rootDir
c.blzOpts = append(c.blzOpts, blobovniczatree.WithRootPath(filepath.Join(rootDir, blobovniczaDir)))
}
}
// WithRootPerm returns option to set permission
// bits of the fs tree.
func WithRootPerm(perm fs.FileMode) Option {
return func(c *cfg) {
c.fsTreeInfo.Permissions = perm
c.blzOpts = append(c.blzOpts, blobovniczatree.WithPermissions(perm))
}
}
// WithSmallSizeLimit returns option to set maximum size of
// "small" object.
func WithSmallSizeLimit(lim uint64) Option {
return func(c *cfg) {
c.smallSizeLimit = lim
c.blzOpts = append(c.blzOpts, blobovniczatree.WithObjectSizeLimit(lim))
}
}
// WithLogger returns option to specify BlobStor's logger.
func WithLogger(l *logger.Logger) Option {
return func(c *cfg) {
c.log = l.With(zap.String("component", "BlobStor"))
c.blzOpts = append(c.blzOpts, blobovniczatree.WithLogger(c.log))
}
}
// WithBlobovniczaShallowDepth returns option to specify
// depth of blobovnicza directories.
func WithBlobovniczaShallowDepth(d uint64) Option {
return func(c *cfg) {
c.blzOpts = append(c.blzOpts, blobovniczatree.WithBlobovniczaShallowDepth(d))
}
}
// WithBlobovniczaShallowWidth returns option to specify
// width of blobovnicza directories.
func WithBlobovniczaShallowWidth(w uint64) Option {
return func(c *cfg) {
c.blzOpts = append(c.blzOpts, blobovniczatree.WithBlobovniczaShallowWidth(w))
}
}
// WithBlobovniczaOpenedCacheSize return option to specify
// maximum number of opened non-active blobovnicza's.
func WithBlobovniczaOpenedCacheSize(sz int) Option {
return func(c *cfg) {
c.blzOpts = append(c.blzOpts, blobovniczatree.WithOpenedCacheSize(sz))
}
}
// WithBlobovniczaSize returns option to specify maximum volume
// of each blobovnicza.
func WithBlobovniczaSize(sz uint64) Option {
return func(c *cfg) {
c.blzOpts = append(c.blzOpts, blobovniczatree.WithBlobovniczaSize(sz))
}
}

49
pkg/local_object_storage/blobstor/blobstor_test.go
View file

@ -2,14 +2,35 @@ package blobstor
import (
"os"
"path/filepath"
"testing"
"github.com/nspcc-dev/neofs-node/pkg/core/object"
"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor/blobovniczatree"
"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor/common"
"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor/fstree"
objectSDK "github.com/nspcc-dev/neofs-sdk-go/object"
"github.com/stretchr/testify/require"
)
const blobovniczaDir = "blobovniczas"
func defaultStorages(p string, smallSizeLimit uint64) []SubStorage {
return []SubStorage{
{
Storage: blobovniczatree.NewBlobovniczaTree(
blobovniczatree.WithRootPath(filepath.Join(p, "blobovniczas")),
blobovniczatree.WithBlobovniczaShallowWidth(1)), // default width is 16, slow init
Policy: func(_ *objectSDK.Object, data []byte) bool {
return uint64(len(data)) <= smallSizeLimit
},
},
{
Storage: fstree.New(fstree.WithPath(p)),
},
}
}
func TestCompression(t *testing.T) {
dir, err := os.MkdirTemp("", "neofs*")
require.NoError(t, err)
@ -21,10 +42,9 @@ func TestCompression(t *testing.T) {
)
newBlobStor := func(t *testing.T, compress bool) *BlobStor {
bs := New(WithCompressObjects(compress),
WithRootPath(dir),
WithSmallSizeLimit(smallSizeLimit),
WithBlobovniczaShallowWidth(1)) // default width is 16, slow init
bs := New(
WithCompressObjects(compress),
WithStorages(defaultStorages(dir, smallSizeLimit)))
require.NoError(t, bs.Open(false))
require.NoError(t, bs.Init())
return bs
@ -86,11 +106,22 @@ func TestBlobstor_needsCompression(t *testing.T) {
require.NoError(t, err)
t.Cleanup(func() { _ = os.RemoveAll(dir) })
bs := New(WithCompressObjects(compress),
WithRootPath(dir),
WithSmallSizeLimit(smallSizeLimit),
WithBlobovniczaShallowWidth(1),
WithUncompressableContentTypes(ct))
bs := New(
WithCompressObjects(compress),
WithUncompressableContentTypes(ct),
WithStorages([]SubStorage{
{
Storage: blobovniczatree.NewBlobovniczaTree(
blobovniczatree.WithRootPath(filepath.Join(dir, "blobovnicza")),
blobovniczatree.WithBlobovniczaShallowWidth(1)), // default width is 16, slow init
Policy: func(_ *objectSDK.Object, data []byte) bool {
return uint64(len(data)) < smallSizeLimit
},
},
{
Storage: fstree.New(fstree.WithPath(dir)),
},
}))
require.NoError(t, bs.Open(false))
require.NoError(t, bs.Init())
return bs

3
pkg/local_object_storage/blobstor/common/storage.go
View file

@ -1,5 +1,7 @@
package common
import "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor/compression"
// Storage represents key-value object storage.
// It is used as a building block for a blobstor of a shard.
type Storage interface {
@ -8,6 +10,7 @@ type Storage interface {
Close() error
Type() string
SetCompressor(cc *compression.CConfig)
Get(GetPrm) (GetRes, error)
GetRange(GetRangePrm) (GetRangeRes, error)

9
pkg/local_object_storage/blobstor/control.go
View file

@ -31,6 +31,10 @@ var ErrInitBlobovniczas = errors.New("failure on blobovnicza initialization stag
func (b *BlobStor) Init() error {
b.log.Debug("initializing...")
if err := b.CConfig.Init(); err != nil {
return err
}
for i := range b.storage {
err := b.storage[i].Storage.Init()
if err != nil {
@ -55,5 +59,10 @@ func (b *BlobStor) Close() error {
continue
}
}
err := b.CConfig.Close()
if firstErr == nil {
firstErr = err
}
return firstErr
}

2
pkg/local_object_storage/blobstor/delete.go
View file

@ -26,7 +26,7 @@ func (b *BlobStor) Delete(prm common.DeletePrm) (common.DeleteRes, error) {
}
}
if len(prm.StorageID) == 0 {
return b.storage[1].Storage.Delete(prm)
return b.storage[len(b.storage)-1].Storage.Delete(prm)
}
return b.storage[0].Storage.Delete(prm)
}

7
pkg/local_object_storage/blobstor/exists_test.go
View file

@ -20,9 +20,8 @@ func TestExists(t *testing.T) {
const smallSizeLimit = 512
b := New(WithRootPath(dir),
WithSmallSizeLimit(smallSizeLimit),
WithBlobovniczaShallowWidth(1)) // default width is 16, slow init
b := New(
WithStorages(defaultStorages(dir, smallSizeLimit)))
require.NoError(t, b.Open(false))
require.NoError(t, b.Init())
@ -65,7 +64,7 @@ func TestExists(t *testing.T) {
require.NotEmpty(t, bigDir)
require.NoError(t, os.Chmod(dir, 0))
t.Cleanup(func() { require.NoError(t, os.Chmod(dir, b.fsTreeInfo.Permissions)) })
t.Cleanup(func() { require.NoError(t, os.Chmod(dir, 0777)) })
// Object exists, first error is logged.
prm.Address = objectCore.AddressOf(objects[0])

8
pkg/local_object_storage/blobstor/fstree/control.go
View file

@ -1,10 +1,16 @@
package fstree
import (
"github.com/nspcc-dev/neofs-node/pkg/util"
)
// Open implements common.Storage.
func (*FSTree) Open(bool) error { return nil }
// Init implements common.Storage.
func (*FSTree) Init() error { return nil }
func (t *FSTree) Init() error {
return util.MkdirAllX(t.RootPath, t.Permissions)
}
// Close implements common.Storage.
func (*FSTree) Close() error { return nil }

22
pkg/local_object_storage/blobstor/fstree/fstree.go
View file

@ -45,6 +45,23 @@ const (
var _ common.Storage = (*FSTree)(nil)
func New(opts ...Option) *FSTree {
f := &FSTree{
Info: Info{
Permissions: 0700,
RootPath: "./",
},
CConfig: nil,
Depth: 4,
DirNameLen: DirNameLen,
}
for i := range opts {
opts[i](f)
}
return f
}
func stringifyAddress(addr oid.Address) string {
return addr.Object().EncodeToString() + "." + addr.Container().EncodeToString()
}
@ -299,3 +316,8 @@ func (t *FSTree) NumberOfObjects() (uint64, error) {
func (*FSTree) Type() string {
return "fstree"
}
// SetCompressor implements common.Storage.
func (t *FSTree) SetCompressor(cc *compression.CConfig) {
t.CConfig = cc
}

31
pkg/local_object_storage/blobstor/fstree/option.go Normal file
View file

@ -0,0 +1,31 @@
package fstree
import (
"io/fs"
)
type Option func(*FSTree)
func WithDepth(d int) Option {
return func(f *FSTree) {
f.Depth = d
}
}
func WithDirNameLen(l int) Option {
return func(f *FSTree) {
f.DirNameLen = l
}
}
func WithPerm(p fs.FileMode) Option {
return func(f *FSTree) {
f.Permissions = p
}
}
func WithPath(p string) Option {
return func(f *FSTree) {
f.RootPath = p
}
}

2
pkg/local_object_storage/blobstor/get.go
View file

@ -23,7 +23,7 @@ func (b *BlobStor) Get(prm common.GetPrm) (common.GetRes, error) {
return common.GetRes{}, errNotFound
}
if len(prm.StorageID) == 0 {
return b.storage[1].Storage.Get(prm)
return b.storage[len(b.storage)-1].Storage.Get(prm)
}
return b.storage[0].Storage.Get(prm)
}

2
pkg/local_object_storage/blobstor/get_range.go
View file

@ -23,7 +23,7 @@ func (b *BlobStor) GetRange(prm common.GetRangePrm) (common.GetRangeRes, error)
return common.GetRangeRes{}, errNotFound
}
if len(prm.StorageID) == 0 {
return b.storage[1].Storage.GetRange(prm)
return b.storage[len(b.storage)-1].Storage.GetRange(prm)
}
return b.storage[0].Storage.GetRange(prm)
}

7
pkg/local_object_storage/blobstor/info.go
View file

@ -4,5 +4,10 @@ import "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor/fstree
// DumpInfo returns information about blob stor.
func (b *BlobStor) DumpInfo() fstree.Info {
return b.cfg.fsTreeInfo
for i := range b.storage {
if b.storage[i].Storage.Type() == "fstree" {
return b.storage[i].Storage.(*fstree.FSTree).Info
}
}
return fstree.Info{}
}

2
pkg/local_object_storage/blobstor/iterate.go
View file

@ -19,7 +19,7 @@ func (b *BlobStor) Iterate(prm common.IteratePrm) (common.IterateRes, error) {
for i := range b.storage {
_, err := b.storage[i].Storage.Iterate(prm)
if err != nil && !prm.IgnoreErrors {
return common.IterateRes{}, fmt.Errorf("blobovnizas iterator failure: %w", err)
return common.IterateRes{}, fmt.Errorf("blobstor iterator failure: %w", err)
}
}
return common.IterateRes{}, nil

5
pkg/local_object_storage/blobstor/iterate_test.go
View file

@ -18,11 +18,8 @@ func TestIterateObjects(t *testing.T) {
// create BlobStor instance
blobStor := New(
WithStorages(defaultStorages(p, smalSz)),
WithCompressObjects(true),
WithRootPath(p),
WithSmallSizeLimit(smalSz),
WithBlobovniczaShallowWidth(1),
WithBlobovniczaShallowDepth(1),
)
defer os.RemoveAll(p)

2
pkg/local_object_storage/blobstor/put.go
View file

@ -33,7 +33,7 @@ func (b *BlobStor) Put(prm common.PutPrm) (common.PutRes, error) {
}
for i := range b.storage {
if b.storage[i].Policy(prm.Object, prm.RawData) {
if b.storage[i].Policy == nil || b.storage[i].Policy(prm.Object, prm.RawData) {
res, err := b.storage[i].Storage.Put(prm)
if err == nil {
storagelog.Write(b.log,

37
pkg/local_object_storage/engine/engine_test.go
View file

@ -8,6 +8,8 @@ import (
"testing"
"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor"
"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor/blobovniczatree"
"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor/fstree"
meta "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/metabase"
"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/pilorama"
"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/shard"
@ -94,6 +96,26 @@ func testNewEngineWithShards(shards ...*shard.Shard) *StorageEngine {
return engine
}
func newStorages(root string, smallSize uint64) []blobstor.SubStorage {
return []blobstor.SubStorage{
{
Storage: blobovniczatree.NewBlobovniczaTree(
blobovniczatree.WithRootPath(filepath.Join(root, "blobovnicza")),
blobovniczatree.WithBlobovniczaShallowDepth(1),
blobovniczatree.WithBlobovniczaShallowWidth(1),
blobovniczatree.WithPermissions(0700)),
Policy: func(_ *object.Object, data []byte) bool {
return uint64(len(data)) < smallSize
},
},
{
Storage: fstree.New(
fstree.WithPath(root),
fstree.WithDepth(1)),
},
}
}
func testNewShard(t testing.TB, id int) *shard.Shard {
sid, err := generateShardID()
require.NoError(t, err)
@ -102,11 +124,9 @@ func testNewShard(t testing.TB, id int) *shard.Shard {
shard.WithID(sid),
shard.WithLogger(zap.L()),
shard.WithBlobStorOptions(
blobstor.WithRootPath(filepath.Join(t.Name(), fmt.Sprintf("%d.blobstor", id))),
blobstor.WithBlobovniczaShallowWidth(2),
blobstor.WithBlobovniczaShallowDepth(2),
blobstor.WithRootPerm(0700),
),
blobstor.WithStorages(
newStorages(filepath.Join(t.Name(), fmt.Sprintf("%d.blobstor", id)),
1<<20))),
shard.WithPiloramaOptions(pilorama.WithPath(filepath.Join(t.Name(), fmt.Sprintf("%d.pilorama", id)))),
shard.WithMetaBaseOptions(
meta.WithPath(filepath.Join(t.Name(), fmt.Sprintf("%d.metabase", id))),
@ -125,10 +145,9 @@ func testEngineFromShardOpts(t *testing.T, num int, extraOpts func(int) []shard.
for i := 0; i < num; i++ {
_, err := engine.AddShard(append([]shard.Option{
shard.WithBlobStorOptions(
blobstor.WithRootPath(filepath.Join(t.Name(), fmt.Sprintf("blobstor%d", i))),
blobstor.WithBlobovniczaShallowWidth(1),
blobstor.WithBlobovniczaShallowDepth(1),
blobstor.WithRootPerm(0700),
blobstor.WithStorages(
newStorages(filepath.Join(t.Name(), fmt.Sprintf("blobstor%d", i)),
1<<20)),
),
shard.WithMetaBaseOptions(
meta.WithPath(filepath.Join(t.Name(), fmt.Sprintf("metabase%d", i))),

7
pkg/local_object_storage/engine/error_test.go
View file

@ -43,12 +43,7 @@ func newEngineWithErrorThreshold(t testing.TB, dir string, errThreshold uint32)
ids[i], err = e.AddShard(
shard.WithLogger(zaptest.NewLogger(t)),
shard.WithBlobStorOptions(
blobstor.WithRootPath(filepath.Join(dir, strconv.Itoa(i))),
blobstor.WithShallowDepth(1),
blobstor.WithBlobovniczaShallowWidth(1),
blobstor.WithBlobovniczaShallowDepth(1),
blobstor.WithSmallSizeLimit(errSmallSize),
blobstor.WithRootPerm(0700)),
blobstor.WithStorages(newStorages(filepath.Join(dir, strconv.Itoa(i)), errSmallSize))),
shard.WithMetaBaseOptions(
meta.WithPath(filepath.Join(dir, fmt.Sprintf("%d.metabase", i))),
meta.WithPermissions(0700),

37
pkg/local_object_storage/shard/control_test.go
View file

@ -39,11 +39,14 @@ func TestShardOpen(t *testing.T) {
return New(
WithLogger(zaptest.NewLogger(t)),
WithBlobStorOptions(
blobstor.WithRootPath(filepath.Join(dir, "blob")),
blobstor.WithShallowDepth(1),
blobstor.WithSmallSizeLimit(1),
blobstor.WithBlobovniczaShallowWidth(1),
blobstor.WithBlobovniczaShallowDepth(1)),
blobstor.WithStorages([]blobstor.SubStorage{
{
Storage: fstree.New(
fstree.WithDirNameLen(2),
fstree.WithPath(filepath.Join(dir, "blob")),
fstree.WithDepth(1)),
},
})),
WithMetaBaseOptions(meta.WithPath(metaPath), meta.WithEpochState(epochState{})),
WithPiloramaOptions(
pilorama.WithPath(filepath.Join(dir, "pilorama"))),
@ -81,11 +84,15 @@ func TestRefillMetabaseCorrupted(t *testing.T) {
dir := t.TempDir()
blobOpts := []blobstor.Option{
blobstor.WithRootPath(filepath.Join(dir, "blob")),
blobstor.WithShallowDepth(1),
blobstor.WithSmallSizeLimit(1),
blobstor.WithBlobovniczaShallowWidth(1),
blobstor.WithBlobovniczaShallowDepth(1)}
blobstor.WithStorages([]blobstor.SubStorage{
{
Storage: fstree.New(
fstree.WithDirNameLen(2),
fstree.WithPath(filepath.Join(dir, "blob")),
fstree.WithDepth(1)),
},
}),
}
sh := New(
WithBlobStorOptions(blobOpts...),
@ -134,9 +141,13 @@ func TestRefillMetabase(t *testing.T) {
defer os.RemoveAll(p)
blobOpts := []blobstor.Option{
blobstor.WithRootPath(filepath.Join(p, "blob")),
blobstor.WithBlobovniczaShallowWidth(1),
blobstor.WithBlobovniczaShallowDepth(1),
blobstor.WithStorages([]blobstor.SubStorage{
{
Storage: fstree.New(
fstree.WithPath(filepath.Join(p, "blob")),
fstree.WithDepth(1)),
},
}),
}
sh := New(

42
pkg/local_object_storage/shard/dump_test.go
View file

@ -13,6 +13,8 @@ import (
"github.com/nspcc-dev/neofs-node/pkg/core/object"
"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobovnicza"
"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor"
"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor/blobovniczatree"
"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor/fstree"
"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/shard"
"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/shard/mode"
"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/writecache"
@ -52,9 +54,7 @@ func testDump(t *testing.T, objCount int, hasWriteCache bool) {
writecache.WithMaxObjectSize(wcBigObjectSize),
writecache.WithLogger(zaptest.NewLogger(t)),
},
[]blobstor.Option{
blobstor.WithLogger(zaptest.NewLogger(t)),
})
nil)
}
defer releaseShard(sh, t)
@ -186,7 +186,7 @@ func testDump(t *testing.T, objCount int, hasWriteCache bool) {
t.Run("skip errors", func(t *testing.T) {
sh := newCustomShard(t, filepath.Join(t.TempDir(), "ignore"), false, nil, nil)
defer releaseShard(sh, t)
t.Cleanup(func() { require.NoError(t, sh.Close()) })
var restorePrm shard.RestorePrm
restorePrm.WithPath(out)
@ -292,14 +292,27 @@ func TestDumpIgnoreErrors(t *testing.T) {
dir := t.TempDir()
bsPath := filepath.Join(dir, "blob")
bsOpts := []blobstor.Option{
blobstor.WithSmallSizeLimit(bsSmallObjectSize),
blobstor.WithRootPath(bsPath),
blobstor.WithCompressObjects(true),
blobstor.WithShallowDepth(1),
blobstor.WithBlobovniczaShallowDepth(1),
blobstor.WithBlobovniczaShallowWidth(2),
blobstor.WithBlobovniczaOpenedCacheSize(1),
bsOpts := func(sw uint64) []blobstor.Option {
return []blobstor.Option{
blobstor.WithCompressObjects(true),
blobstor.WithStorages([]blobstor.SubStorage{
{
Storage: blobovniczatree.NewBlobovniczaTree(
blobovniczatree.WithRootPath(filepath.Join(bsPath, "blobovnicza")),
blobovniczatree.WithBlobovniczaShallowDepth(1),
blobovniczatree.WithBlobovniczaShallowWidth(sw),
blobovniczatree.WithOpenedCacheSize(1)),
Policy: func(_ *objectSDK.Object, data []byte) bool {
return len(data) < bsSmallObjectSize
},
},
{
Storage: fstree.New(
fstree.WithPath(bsPath),
fstree.WithDepth(1)),
},
}),
}
}
wcPath := filepath.Join(dir, "writecache")
wcOpts := []writecache.Option{
@ -307,7 +320,7 @@ func TestDumpIgnoreErrors(t *testing.T) {
writecache.WithSmallObjectSize(wcSmallObjectSize),
writecache.WithMaxObjectSize(wcBigObjectSize),
}
sh := newCustomShard(t, dir, true, wcOpts, bsOpts)
sh := newCustomShard(t, dir, true, wcOpts, bsOpts(2))
objects := make([]*objectSDK.Object, objCount)
for i := 0; i < objCount; i++ {
@ -355,8 +368,7 @@ func TestDumpIgnoreErrors(t *testing.T) {
require.NoError(t, os.MkdirAll(filepath.Join(bsPath, "ZZ"), 0))
}
bsOpts = append(bsOpts, blobstor.WithBlobovniczaShallowWidth(3))
sh = newCustomShard(t, dir, true, wcOpts, bsOpts)
sh = newCustomShard(t, dir, true, wcOpts, bsOpts(3))
require.NoError(t, sh.SetMode(mode.ReadOnly))
{

22
pkg/local_object_storage/shard/lock_test.go
View file

@ -7,6 +7,8 @@ import (
objectcore "github.com/nspcc-dev/neofs-node/pkg/core/object"
"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor"
"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor/blobovniczatree"
"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor/fstree"
meta "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/metabase"
"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/shard"
apistatus "github.com/nspcc-dev/neofs-sdk-go/client/status"
@ -24,11 +26,21 @@ func TestShard_Lock(t *testing.T) {
opts := []shard.Option{
shard.WithLogger(zap.NewNop()),
shard.WithBlobStorOptions(
[]blobstor.Option{
blobstor.WithRootPath(filepath.Join(rootPath, "blob")),
blobstor.WithBlobovniczaShallowWidth(2),
blobstor.WithBlobovniczaShallowDepth(2),
}...,
blobstor.WithStorages([]blobstor.SubStorage{
{
Storage: blobovniczatree.NewBlobovniczaTree(
blobovniczatree.WithRootPath(filepath.Join(rootPath, "blob", "blobovnicza")),
blobovniczatree.WithBlobovniczaShallowDepth(2),
blobovniczatree.WithBlobovniczaShallowWidth(2)),
Policy: func(_ *object.Object, data []byte) bool {
return len(data) <= 1<<20
},
},
{
Storage: fstree.New(
fstree.WithPath(filepath.Join(rootPath, "blob"))),
},
}),
),
shard.WithMetaBaseOptions(
meta.WithPath(filepath.Join(rootPath, "meta")),

21
pkg/local_object_storage/shard/range_test.go
View file

@ -2,16 +2,20 @@ package shard_test
import (
"math"
"path/filepath"
"testing"
"github.com/nspcc-dev/neo-go/pkg/util/slice"
"github.com/nspcc-dev/neofs-node/pkg/core/object"
"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor"
"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor/blobovniczatree"
"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor/fstree"
"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/shard"
"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/writecache"
apistatus "github.com/nspcc-dev/neofs-sdk-go/client/status"
objectSDK "github.com/nspcc-dev/neofs-sdk-go/object"
"github.com/stretchr/testify/require"
"go.uber.org/zap/zaptest"
)
func TestShard_GetRange(t *testing.T) {
@ -62,7 +66,22 @@ func testShardGetRange(t *testing.T, hasWriteCache bool) {
sh := newCustomShard(t, t.TempDir(), hasWriteCache,
[]writecache.Option{writecache.WithMaxMemSize(0), writecache.WithMaxObjectSize(writeCacheMaxSize)},
[]blobstor.Option{blobstor.WithSmallSizeLimit(smallObjectSize)})
[]blobstor.Option{blobstor.WithStorages([]blobstor.SubStorage{
{
Storage: blobovniczatree.NewBlobovniczaTree(
blobovniczatree.WithLogger(zaptest.NewLogger(t)),
blobovniczatree.WithRootPath(filepath.Join(t.TempDir(), "blob", "blobovnicza")),
blobovniczatree.WithBlobovniczaShallowDepth(1),
blobovniczatree.WithBlobovniczaShallowWidth(1)),
Policy: func(_ *objectSDK.Object, data []byte) bool {
return len(data) <= smallObjectSize
},
},
{
Storage: fstree.New(
fstree.WithPath(filepath.Join(t.TempDir(), "blob"))),
},
})})
defer releaseShard(sh, t)
for _, tc := range testCases {

38
pkg/local_object_storage/shard/shard_test.go
View file

@ -4,12 +4,12 @@ import (
"crypto/sha256"
"math"
"math/rand"
"os"
"path/filepath"
"strings"
"testing"
"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor"
"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor/blobovniczatree"
"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor/fstree"
meta "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/metabase"
"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/pilorama"
"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/shard"
@ -24,6 +24,7 @@ import (
"github.com/nspcc-dev/tzhash/tz"
"github.com/stretchr/testify/require"
"go.uber.org/zap"
"go.uber.org/zap/zaptest"
)
type epochState struct{}
@ -45,15 +46,31 @@ func newCustomShard(t testing.TB, rootPath string, enableWriteCache bool, wcOpts
rootPath = filepath.Join(rootPath, "nowc")
}
if bsOpts == nil {
bsOpts = []blobstor.Option{
blobstor.WithLogger(zaptest.NewLogger(t)),
blobstor.WithStorages([]blobstor.SubStorage{
{
Storage: blobovniczatree.NewBlobovniczaTree(
blobovniczatree.WithLogger(zaptest.NewLogger(t)),
blobovniczatree.WithRootPath(filepath.Join(rootPath, "blob", "blobovnicza")),
blobovniczatree.WithBlobovniczaShallowDepth(1),
blobovniczatree.WithBlobovniczaShallowWidth(1)),
Policy: func(_ *object.Object, data []byte) bool {
return len(data) <= 1<<20
},
},
{
Storage: fstree.New(
fstree.WithPath(filepath.Join(rootPath, "blob"))),
},
}),
}
}
opts := []shard.Option{
shard.WithLogger(zap.L()),
shard.WithBlobStorOptions(
append([]blobstor.Option{
blobstor.WithRootPath(filepath.Join(rootPath, "blob")),
blobstor.WithBlobovniczaShallowWidth(2),
blobstor.WithBlobovniczaShallowDepth(2),
}, bsOpts...)...,
),
shard.WithBlobStorOptions(bsOpts...),
shard.WithMetaBaseOptions(
meta.WithPath(filepath.Join(rootPath, "meta")),
meta.WithEpochState(epochState{}),
@ -76,8 +93,7 @@ func newCustomShard(t testing.TB, rootPath string, enableWriteCache bool, wcOpts
}
func releaseShard(s *shard.Shard, t testing.TB) {
s.Close()
os.RemoveAll(strings.Split(t.Name(), string(os.PathSeparator))[0])
require.NoError(t, s.Close())
}
func generateObject(t *testing.T) *object.Object {

1
pkg/local_object_storage/writecache/mode.go
View file

@ -27,7 +27,6 @@ func (c *cache) SetMode(m mode.Mode) error {
if err := c.db.Close(); err != nil {
return fmt.Errorf("can't close write-cache database: %w", err)
}
c.db = nil
}
// Suspend producers to ensure there are channel send operations in fly.