diff --git a/cmd/neofs-node/config.go b/cmd/neofs-node/config.go index 0cc276b35..dc9c29bfa 100644 --- a/cmd/neofs-node/config.go +++ b/cmd/neofs-node/config.go @@ -15,6 +15,8 @@ import ( contractsconfig "github.com/nspcc-dev/neofs-node/cmd/neofs-node/config/contracts" engineconfig "github.com/nspcc-dev/neofs-node/cmd/neofs-node/config/engine" shardconfig "github.com/nspcc-dev/neofs-node/cmd/neofs-node/config/engine/shard" + blobovniczaconfig "github.com/nspcc-dev/neofs-node/cmd/neofs-node/config/engine/shard/blobstor/blobovnicza" + fstreeconfig "github.com/nspcc-dev/neofs-node/cmd/neofs-node/config/engine/shard/blobstor/fstree" loggerconfig "github.com/nspcc-dev/neofs-node/cmd/neofs-node/config/logger" metricsconfig "github.com/nspcc-dev/neofs-node/cmd/neofs-node/config/metrics" nodeconfig "github.com/nspcc-dev/neofs-node/cmd/neofs-node/config/node" @@ -22,6 +24,8 @@ import ( "github.com/nspcc-dev/neofs-node/pkg/core/container" netmapCore "github.com/nspcc-dev/neofs-node/pkg/core/netmap" "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor" + "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor/blobovniczatree" + "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor/fstree" "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/engine" meta "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/metabase" "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/pilorama" @@ -47,6 +51,7 @@ import ( "github.com/nspcc-dev/neofs-node/pkg/util/logger" "github.com/nspcc-dev/neofs-node/pkg/util/state" "github.com/nspcc-dev/neofs-sdk-go/netmap" + objectSDK "github.com/nspcc-dev/neofs-sdk-go/object" "github.com/nspcc-dev/neofs-sdk-go/user" "github.com/nspcc-dev/neofs-sdk-go/version" "github.com/panjf2000/ants/v2" @@ -418,7 +423,7 @@ func initShardOptions(c *cfg) { } blobStorCfg := sc.BlobStor() - blobovniczaCfg := blobStorCfg.Blobovnicza() + storages := blobStorCfg.Storages() metabaseCfg := sc.Metabase() gcCfg := sc.GC() @@ -426,19 +431,47 @@ func initShardOptions(c *cfg) { piloramaCfg := sc.Pilorama() if config.BoolSafe(c.appCfg.Sub("tree"), "enabled") { - piloramaPath := piloramaCfg.Path() - if piloramaPath == "" { - piloramaPath = filepath.Join(blobStorCfg.Path(), "pilorama.db") - } - piloramaOpts = []pilorama.Option{ - pilorama.WithPath(piloramaPath), + pilorama.WithPath(piloramaCfg.Path()), pilorama.WithPerm(piloramaCfg.Perm()), pilorama.WithNoSync(piloramaCfg.NoSync()), pilorama.WithMaxBatchSize(piloramaCfg.MaxBatchSize()), pilorama.WithMaxBatchDelay(piloramaCfg.MaxBatchDelay())} } + var st []blobstor.SubStorage + for i := range storages { + switch storages[i].Type() { + case "blobovniczas": + sub := blobovniczaconfig.From((*config.Config)(storages[i])) + lim := sc.SmallSizeLimit() + st = append(st, blobstor.SubStorage{ + Storage: blobovniczatree.NewBlobovniczaTree( + blobovniczatree.WithLogger(c.log), + blobovniczatree.WithRootPath(storages[i].Path()), + blobovniczatree.WithPermissions(storages[i].Perm()), + blobovniczatree.WithBlobovniczaSize(sub.Size()), + blobovniczatree.WithBlobovniczaShallowDepth(sub.ShallowDepth()), + blobovniczatree.WithBlobovniczaShallowWidth(sub.ShallowWidth()), + blobovniczatree.WithOpenedCacheSize(sub.OpenedCacheSize())), + Policy: func(_ *objectSDK.Object, data []byte) bool { + return uint64(len(data)) < lim + }, + }) + case "fstree": + sub := fstreeconfig.From((*config.Config)(storages[i])) + st = append(st, blobstor.SubStorage{ + Storage: fstree.New( + fstree.WithPath(storages[i].Path()), + fstree.WithPerm(storages[i].Perm()), + fstree.WithDepth(sub.Depth())), + Policy: func(_ *objectSDK.Object, data []byte) bool { + return true + }, + }) + } + } + metaPath := metabaseCfg.Path() metaPerm := metabaseCfg.BoltDB().Perm() fatalOnErr(util.MkdirAllX(filepath.Dir(metaPath), metaPerm)) @@ -453,15 +486,9 @@ func initShardOptions(c *cfg) { shard.WithRefillMetabase(sc.RefillMetabase()), shard.WithMode(sc.Mode()), shard.WithBlobStorOptions( - blobstor.WithRootPath(blobStorCfg.Path()), - blobstor.WithCompressObjects(blobStorCfg.Compress()), - blobstor.WithRootPerm(blobStorCfg.Perm()), - blobstor.WithShallowDepth(blobStorCfg.ShallowDepth()), - blobstor.WithSmallSizeLimit(blobStorCfg.SmallSizeLimit()), - blobstor.WithBlobovniczaSize(blobovniczaCfg.Size()), - blobstor.WithBlobovniczaShallowDepth(blobovniczaCfg.ShallowDepth()), - blobstor.WithBlobovniczaShallowWidth(blobovniczaCfg.ShallowWidth()), - blobstor.WithBlobovniczaOpenedCacheSize(blobovniczaCfg.OpenedCacheSize()), + blobstor.WithCompressObjects(sc.Compress()), + blobstor.WithUncompressableContentTypes(sc.UncompressableContentTypes()), + blobstor.WithStorages(st), blobstor.WithLogger(c.log), ), shard.WithMetaBaseOptions( diff --git a/cmd/neofs-node/config/engine/config.go b/cmd/neofs-node/config/engine/config.go index 51f8a393e..37b6f1c33 100644 --- a/cmd/neofs-node/config/engine/config.go +++ b/cmd/neofs-node/config/engine/config.go @@ -39,7 +39,7 @@ func IterateShards(c *config.Config, required bool, f func(*shardconfig.Config)) // must have different paths, so if it is missing, the shard is not here. // At the same time checking for "blobstor" section doesn't work proper // with configuration via the environment. - if (*config.Config)(sc).Value("blobstor.path") == nil { + if (*config.Config)(sc).Value("metabase.path") == nil { break } (*config.Config)(sc).SetDefault(def) diff --git a/cmd/neofs-node/config/engine/config_test.go b/cmd/neofs-node/config/engine/config_test.go index 566841408..54f7bb240 100644 --- a/cmd/neofs-node/config/engine/config_test.go +++ b/cmd/neofs-node/config/engine/config_test.go @@ -8,6 +8,8 @@ import ( "github.com/nspcc-dev/neofs-node/cmd/neofs-node/config" engineconfig "github.com/nspcc-dev/neofs-node/cmd/neofs-node/config/engine" shardconfig "github.com/nspcc-dev/neofs-node/cmd/neofs-node/config/engine/shard" + blobovniczaconfig "github.com/nspcc-dev/neofs-node/cmd/neofs-node/config/engine/shard/blobstor/blobovnicza" + fstreeconfig "github.com/nspcc-dev/neofs-node/cmd/neofs-node/config/engine/shard/blobstor/fstree" piloramaconfig "github.com/nspcc-dev/neofs-node/cmd/neofs-node/config/engine/shard/pilorama" configtest "github.com/nspcc-dev/neofs-node/cmd/neofs-node/config/test" "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/shard/mode" @@ -53,7 +55,7 @@ func TestEngineSection(t *testing.T) { wc := sc.WriteCache() meta := sc.Metabase() blob := sc.BlobStor() - blz := blob.Blobovnicza() + ss := blob.Storages() pl := sc.Pilorama() gc := sc.GC() @@ -79,18 +81,23 @@ func TestEngineSection(t *testing.T) { require.Equal(t, 100, meta.BoltDB().MaxBatchSize()) require.Equal(t, 10*time.Millisecond, meta.BoltDB().MaxBatchDelay()) - require.Equal(t, "tmp/0/blob", blob.Path()) - require.EqualValues(t, 0644, blob.Perm()) - require.Equal(t, true, blob.Compress()) - require.Equal(t, []string{"audio/*", "video/*"}, blob.UncompressableContentTypes()) - require.EqualValues(t, 5, blob.ShallowDepth()) - require.EqualValues(t, 102400, blob.SmallSizeLimit()) + require.Equal(t, true, sc.Compress()) + require.Equal(t, []string{"audio/*", "video/*"}, sc.UncompressableContentTypes()) + require.EqualValues(t, 102400, sc.SmallSizeLimit()) + require.Equal(t, 2, len(ss)) + blz := blobovniczaconfig.From((*config.Config)(ss[0])) + require.Equal(t, "tmp/0/blob/blobovnicza", ss[0].Path()) + require.EqualValues(t, 0644, blz.BoltDB().Perm()) require.EqualValues(t, 4194304, blz.Size()) require.EqualValues(t, 1, blz.ShallowDepth()) require.EqualValues(t, 4, blz.ShallowWidth()) require.EqualValues(t, 50, blz.OpenedCacheSize()) + require.Equal(t, "tmp/0/blob", ss[1].Path()) + require.EqualValues(t, 0644, ss[1].Perm()) + require.EqualValues(t, 5, fstreeconfig.From((*config.Config)(ss[1])).Depth()) + require.EqualValues(t, 150, gc.RemoverBatchSize()) require.Equal(t, 2*time.Minute, gc.RemoverSleepInterval()) @@ -117,18 +124,23 @@ func TestEngineSection(t *testing.T) { require.Equal(t, 200, meta.BoltDB().MaxBatchSize()) require.Equal(t, 20*time.Millisecond, meta.BoltDB().MaxBatchDelay()) - require.Equal(t, "tmp/1/blob", blob.Path()) - require.EqualValues(t, 0644, blob.Perm()) - require.Equal(t, false, blob.Compress()) - require.Equal(t, []string(nil), blob.UncompressableContentTypes()) - require.EqualValues(t, 5, blob.ShallowDepth()) - require.EqualValues(t, 102400, blob.SmallSizeLimit()) + require.Equal(t, false, sc.Compress()) + require.Equal(t, []string(nil), sc.UncompressableContentTypes()) + require.EqualValues(t, 102400, sc.SmallSizeLimit()) + require.Equal(t, 2, len(ss)) + + blz := blobovniczaconfig.From((*config.Config)(ss[0])) + require.Equal(t, "tmp/1/blob/blobovnicza", ss[0].Path()) require.EqualValues(t, 4194304, blz.Size()) require.EqualValues(t, 1, blz.ShallowDepth()) require.EqualValues(t, 4, blz.ShallowWidth()) require.EqualValues(t, 50, blz.OpenedCacheSize()) + require.Equal(t, "tmp/1/blob", ss[1].Path()) + require.EqualValues(t, 0644, ss[1].Perm()) + require.EqualValues(t, 5, fstreeconfig.From((*config.Config)(ss[1])).Depth()) + require.EqualValues(t, 200, gc.RemoverBatchSize()) require.Equal(t, 5*time.Minute, gc.RemoverSleepInterval()) diff --git a/cmd/neofs-node/config/engine/shard/blobstor/blobovnicza/config.go b/cmd/neofs-node/config/engine/shard/blobstor/blobovnicza/config.go index 1cb571b91..89edc5a09 100644 --- a/cmd/neofs-node/config/engine/shard/blobstor/blobovnicza/config.go +++ b/cmd/neofs-node/config/engine/shard/blobstor/blobovnicza/config.go @@ -29,6 +29,11 @@ func From(c *config.Config) *Config { return (*Config)(c) } +// Type returns the storage type. +func (x *Config) Type() string { + return "blobovnicza" +} + // Size returns the value of "size" config parameter. // // Returns SizeDefault if the value is not a positive number. diff --git a/cmd/neofs-node/config/engine/shard/blobstor/config.go b/cmd/neofs-node/config/engine/shard/blobstor/config.go index 524f8ad75..e333e8857 100644 --- a/cmd/neofs-node/config/engine/shard/blobstor/config.go +++ b/cmd/neofs-node/config/engine/shard/blobstor/config.go @@ -1,122 +1,36 @@ package blobstorconfig import ( - "io/fs" + "strconv" "github.com/nspcc-dev/neofs-node/cmd/neofs-node/config" - blobovniczaconfig "github.com/nspcc-dev/neofs-node/cmd/neofs-node/config/engine/shard/blobstor/blobovnicza" - "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor/fstree" + "github.com/nspcc-dev/neofs-node/cmd/neofs-node/config/engine/shard/blobstor/storage" ) // Config is a wrapper over the config section // which provides access to BlobStor configurations. type Config config.Config -// config defaults -const ( - // PermDefault are default permission bits for BlobStor data. - PermDefault = 0660 - - // ShallowDepthDefault is a default shallow dir depth. - ShallowDepthDefault = 4 - - // SmallSizeLimitDefault is a default limit of small objects payload in bytes. - SmallSizeLimitDefault = 1 << 20 -) - // From wraps config section into Config. func From(c *config.Config) *Config { return (*Config)(c) } -// Path returns the value of "path" config parameter. -// -// Panics if the value is not a non-empty string. -func (x *Config) Path() string { - p := config.String( - (*config.Config)(x), - "path", - ) - - if p == "" { - panic("blobstor path not set") +// Storages returns the value of storage subcomponents. +func (x *Config) Storages() []*storage.Config { + var ss []*storage.Config + for i := 0; ; i++ { + typ := config.String( + (*config.Config)(x), + strconv.Itoa(i)+".type") + switch typ { + case "": + return ss + case "fstree", "blobovnicza": + sub := storage.From((*config.Config)(x).Sub(strconv.Itoa(i))) + ss = append(ss, sub) + default: + panic("invalid type") + } } - - return p -} - -// Perm returns the value of "perm" config parameter as a fs.FileMode. -// -// Returns PermDefault if the value is not a non-zero number. -func (x *Config) Perm() fs.FileMode { - p := config.UintSafe( - (*config.Config)(x), - "perm", - ) - - if p == 0 { - p = PermDefault - } - - return fs.FileMode(p) -} - -// ShallowDepth returns the value of "depth" config parameter. -// -// Returns ShallowDepthDefault if the value is out of -// [1:fstree.MaxDepth] range. -func (x *Config) ShallowDepth() int { - d := config.IntSafe( - (*config.Config)(x), - "depth", - ) - - if d >= 1 && d <= fstree.MaxDepth { - return int(d) - } - - return ShallowDepthDefault -} - -// Compress returns the value of "compress" config parameter. -// -// Returns false if the value is not a valid bool. -func (x *Config) Compress() bool { - return config.BoolSafe( - (*config.Config)(x), - "compress", - ) -} - -// UncompressableContentTypes returns the value of "compress_skip_content_types" config parameter. -// -// Returns nil if a the value is missing or is invalid. -func (x *Config) UncompressableContentTypes() []string { - return config.StringSliceSafe( - (*config.Config)(x), - "compression_exclude_content_types") -} - -// SmallSizeLimit returns the value of "small_object_size" config parameter. -// -// Returns SmallSizeLimitDefault if the value is not a positive number. -func (x *Config) SmallSizeLimit() uint64 { - l := config.SizeInBytesSafe( - (*config.Config)(x), - "small_object_size", - ) - - if l > 0 { - return l - } - - return SmallSizeLimitDefault -} - -// Blobovnicza returns "blobovnicza" subsection as a blobovniczaconfig.Config. -func (x *Config) Blobovnicza() *blobovniczaconfig.Config { - return blobovniczaconfig.From( - (*config.Config)(x). - Sub("blobovnicza"), - ) } diff --git a/cmd/neofs-node/config/engine/shard/blobstor/fstree/config.go b/cmd/neofs-node/config/engine/shard/blobstor/fstree/config.go new file mode 100644 index 000000000..69526bb17 --- /dev/null +++ b/cmd/neofs-node/config/engine/shard/blobstor/fstree/config.go @@ -0,0 +1,40 @@ +package fstree + +import ( + "github.com/nspcc-dev/neofs-node/cmd/neofs-node/config" + "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor/fstree" +) + +// Config is a wrapper over the config section +// which provides access to Blobovnicza configurations. +type Config config.Config + +// DepthDefault is a default shallow dir depth. +const DepthDefault = 4 + +// From wraps config section into Config. +func From(c *config.Config) *Config { + return (*Config)(c) +} + +// Type returns the storage type. +func (x *Config) Type() string { + return "fstree" +} + +// Depth returns the value of "depth" config parameter. +// +// Returns DepthDefault if the value is out of +// [1:fstree.MaxDepth] range. +func (x *Config) Depth() int { + d := config.IntSafe( + (*config.Config)(x), + "depth", + ) + + if d >= 1 && d <= fstree.MaxDepth { + return int(d) + } + + return DepthDefault +} diff --git a/cmd/neofs-node/config/engine/shard/blobstor/storage/config.go b/cmd/neofs-node/config/engine/shard/blobstor/storage/config.go new file mode 100644 index 000000000..84c957422 --- /dev/null +++ b/cmd/neofs-node/config/engine/shard/blobstor/storage/config.go @@ -0,0 +1,55 @@ +package storage + +import ( + "io/fs" + + "github.com/nspcc-dev/neofs-node/cmd/neofs-node/config" +) + +type Config config.Config + +// PermDefault are default permission bits for BlobStor data. +const PermDefault = 0660 + +func From(x *config.Config) *Config { + return (*Config)(x) +} + +// Type returns storage type. +func (x *Config) Type() string { + return config.String( + (*config.Config)(x), + "type") +} + +// Path returns the value of "path" config parameter. +// +// Panics if the value is not a non-empty string. +func (x *Config) Path() string { + p := config.String( + (*config.Config)(x), + "path", + ) + + if p == "" { + panic("blobstor path not set") + } + + return p +} + +// Perm returns the value of "perm" config parameter as a fs.FileMode. +// +// Returns PermDefault if the value is not a non-zero number. +func (x *Config) Perm() fs.FileMode { + p := config.UintSafe( + (*config.Config)(x), + "perm", + ) + + if p == 0 { + p = PermDefault + } + + return fs.FileMode(p) +} diff --git a/cmd/neofs-node/config/engine/shard/config.go b/cmd/neofs-node/config/engine/shard/config.go index 77c032a53..ab1641ae2 100644 --- a/cmd/neofs-node/config/engine/shard/config.go +++ b/cmd/neofs-node/config/engine/shard/config.go @@ -16,11 +16,49 @@ import ( // which provides access to Shard configurations. type Config config.Config +// SmallSizeLimitDefault is a default limit of small objects payload in bytes. +const SmallSizeLimitDefault = 1 << 20 + // From wraps config section into Config. func From(c *config.Config) *Config { return (*Config)(c) } +// Compress returns the value of "compress" config parameter. +// +// Returns false if the value is not a valid bool. +func (x *Config) Compress() bool { + return config.BoolSafe( + (*config.Config)(x), + "compress", + ) +} + +// UncompressableContentTypes returns the value of "compress_skip_content_types" config parameter. +// +// Returns nil if a the value is missing or is invalid. +func (x *Config) UncompressableContentTypes() []string { + return config.StringSliceSafe( + (*config.Config)(x), + "compression_exclude_content_types") +} + +// SmallSizeLimit returns the value of "small_object_size" config parameter. +// +// Returns SmallSizeLimitDefault if the value is not a positive number. +func (x *Config) SmallSizeLimit() uint64 { + l := config.SizeInBytesSafe( + (*config.Config)(x), + "small_object_size", + ) + + if l > 0 { + return l + } + + return SmallSizeLimitDefault +} + // BlobStor returns "blobstor" subsection as a blobstorconfig.Config. func (x *Config) BlobStor() *blobstorconfig.Config { return blobstorconfig.From( diff --git a/config/example/node.env b/config/example/node.env index 59b02e201..1544c444f 100644 --- a/config/example/node.env +++ b/config/example/node.env @@ -101,17 +101,22 @@ NEOFS_STORAGE_SHARD_0_METABASE_PERM=0644 NEOFS_STORAGE_SHARD_0_METABASE_MAX_BATCH_SIZE=100 NEOFS_STORAGE_SHARD_0_METABASE_MAX_BATCH_DELAY=10ms ### Blobstor config -NEOFS_STORAGE_SHARD_0_BLOBSTOR_PATH=tmp/0/blob -NEOFS_STORAGE_SHARD_0_BLOBSTOR_PERM=0644 -NEOFS_STORAGE_SHARD_0_BLOBSTOR_COMPRESS=true -NEOFS_STORAGE_SHARD_0_BLOBSTOR_COMPRESSION_EXCLUDE_CONTENT_TYPES="audio/* video/*" -NEOFS_STORAGE_SHARD_0_BLOBSTOR_DEPTH=5 -NEOFS_STORAGE_SHARD_0_BLOBSTOR_SMALL_OBJECT_SIZE=102400 +NEOFS_STORAGE_SHARD_0_COMPRESS=true +NEOFS_STORAGE_SHARD_0_COMPRESSION_EXCLUDE_CONTENT_TYPES="audio/* video/*" +NEOFS_STORAGE_SHARD_0_SMALL_OBJECT_SIZE=102400 ### Blobovnicza config -NEOFS_STORAGE_SHARD_0_BLOBSTOR_BLOBOVNICZA_SIZE=4194304 -NEOFS_STORAGE_SHARD_0_BLOBSTOR_BLOBOVNICZA_DEPTH=1 -NEOFS_STORAGE_SHARD_0_BLOBSTOR_BLOBOVNICZA_WIDTH=4 -NEOFS_STORAGE_SHARD_0_BLOBSTOR_BLOBOVNICZA_OPENED_CACHE_CAPACITY=50 +NEOFS_STORAGE_SHARD_0_BLOBSTOR_0_PATH=tmp/0/blob/blobovnicza +NEOFS_STORAGE_SHARD_0_BLOBSTOR_0_PERM=0644 +NEOFS_STORAGE_SHARD_0_BLOBSTOR_0_TYPE=blobovnicza +NEOFS_STORAGE_SHARD_0_BLOBSTOR_0_SIZE=4194304 +NEOFS_STORAGE_SHARD_0_BLOBSTOR_0_DEPTH=1 +NEOFS_STORAGE_SHARD_0_BLOBSTOR_0_WIDTH=4 +NEOFS_STORAGE_SHARD_0_BLOBSTOR_0_OPENED_CACHE_CAPACITY=50 +### FSTree config +NEOFS_STORAGE_SHARD_0_BLOBSTOR_1_TYPE=fstree +NEOFS_STORAGE_SHARD_0_BLOBSTOR_1_PATH=tmp/0/blob +NEOFS_STORAGE_SHARD_0_BLOBSTOR_1_PERM=0644 +NEOFS_STORAGE_SHARD_0_BLOBSTOR_1_DEPTH=5 ### Pilorama config NEOFS_STORAGE_SHARD_0_PILORAMA_PATH="tmp/0/blob/pilorama.db" NEOFS_STORAGE_SHARD_0_PILORAMA_MAX_BATCH_DELAY=10ms @@ -141,16 +146,20 @@ NEOFS_STORAGE_SHARD_1_METABASE_PERM=0644 NEOFS_STORAGE_SHARD_1_METABASE_MAX_BATCH_SIZE=200 NEOFS_STORAGE_SHARD_1_METABASE_MAX_BATCH_DELAY=20ms ### Blobstor config -NEOFS_STORAGE_SHARD_1_BLOBSTOR_PATH=tmp/1/blob -NEOFS_STORAGE_SHARD_1_BLOBSTOR_PERM=0644 -NEOFS_STORAGE_SHARD_1_BLOBSTOR_COMPRESS=false -NEOFS_STORAGE_SHARD_1_BLOBSTOR_DEPTH=5 -NEOFS_STORAGE_SHARD_1_BLOBSTOR_SMALL_OBJECT_SIZE=102400 +NEOFS_STORAGE_SHARD_1_COMPRESS=false +NEOFS_STORAGE_SHARD_1_SMALL_OBJECT_SIZE=102400 ### Blobovnicza config -NEOFS_STORAGE_SHARD_1_BLOBSTOR_BLOBOVNICZA_SIZE=4194304 -NEOFS_STORAGE_SHARD_1_BLOBSTOR_BLOBOVNICZA_DEPTH=1 -NEOFS_STORAGE_SHARD_1_BLOBSTOR_BLOBOVNICZA_WIDTH=4 -NEOFS_STORAGE_SHARD_1_BLOBSTOR_BLOBOVNICZA_OPENED_CACHE_CAPACITY=50 +NEOFS_STORAGE_SHARD_1_BLOBSTOR_0_TYPE=blobovnicza +NEOFS_STORAGE_SHARD_1_BLOBSTOR_0_PATH=tmp/1/blob/blobovnicza +NEOFS_STORAGE_SHARD_1_BLOBSTOR_0_SIZE=4194304 +NEOFS_STORAGE_SHARD_1_BLOBSTOR_0_DEPTH=1 +NEOFS_STORAGE_SHARD_1_BLOBSTOR_0_WIDTH=4 +NEOFS_STORAGE_SHARD_1_BLOBSTOR_0_OPENED_CACHE_CAPACITY=50 +### FSTree config +NEOFS_STORAGE_SHARD_1_BLOBSTOR_1_TYPE=fstree +NEOFS_STORAGE_SHARD_1_BLOBSTOR_1_PATH=tmp/1/blob +NEOFS_STORAGE_SHARD_1_BLOBSTOR_1_PERM=0644 +NEOFS_STORAGE_SHARD_1_BLOBSTOR_1_DEPTH=5 ### Pilorama config NEOFS_STORAGE_SHARD_1_PILORAMA_PATH="tmp/1/blob/pilorama.db" NEOFS_STORAGE_SHARD_1_PILORAMA_PERM=0644 diff --git a/config/example/node.json b/config/example/node.json index 6516cd4e9..ab639adbc 100644 --- a/config/example/node.json +++ b/config/example/node.json @@ -147,22 +147,28 @@ "max_batch_size": 100, "max_batch_delay": "10ms" }, - "blobstor": { - "path": "tmp/0/blob", - "perm": "0644", - "compress": true, - "compression_exclude_content_types": [ - "audio/*", "video/*" - ], - "depth": 5, - "small_object_size": 102400, - "blobovnicza": { + "compress": true, + "compression_exclude_content_types": [ + "audio/*", "video/*" + ], + "small_object_size": 102400, + "blobstor": [ + { + "type": "blobovnicza", + "path": "tmp/0/blob/blobovnicza", + "perm": "0644", "size": 4194304, "depth": 1, "width": 4, "opened_cache_capacity": 50 + }, + { + "type": "fstree", + "path": "tmp/0/blob", + "perm": "0644", + "depth": 5 } - }, + ], "pilorama": { "path": "tmp/0/blob/pilorama.db", "max_batch_delay": "10ms", @@ -191,19 +197,25 @@ "max_batch_size": 200, "max_batch_delay": "20ms" }, - "blobstor": { - "path": "tmp/1/blob", - "perm": "0644", - "compress": false, - "depth": 5, - "small_object_size": 102400, - "blobovnicza": { + "compress": false, + "small_object_size": 102400, + "blobstor": [ + { + "type": "blobovnicza", + "path": "tmp/1/blob/blobovnicza", + "perm": "0644", "size": 4194304, "depth": 1, "width": 4, "opened_cache_capacity": 50 + }, + { + "type": "fstree", + "path": "tmp/1/blob", + "perm": "0644", + "depth": 5 } - }, + ], "pilorama": { "path": "tmp/1/blob/pilorama.db", "perm": "0644", diff --git a/config/example/node.yaml b/config/example/node.yaml index 88a2151ca..89180417c 100644 --- a/config/example/node.yaml +++ b/config/example/node.yaml @@ -126,17 +126,17 @@ storage: max_batch_delay: 5ms # maximum delay for a batch of operations to be executed max_batch_size: 100 # maximum amount of operations in a single batch - blobstor: - compress: false # turn on/off zstd(level 3) compression of stored objects - perm: 0644 # permissions for blobstor files(directories: +x for current user and group) - depth: 5 # max depth of object tree storage in FS - small_object_size: 100 kb # size threshold for "small" objects which are cached in key-value DB, not in FS, bytes + compress: false # turn on/off zstd(level 3) compression of stored objects + small_object_size: 100 kb # size threshold for "small" objects which are cached in key-value DB, not in FS, bytes - blobovnicza: - size: 4m # approximate size limit of single blobovnicza instance, total size will be: size*width^(depth+1), bytes + blobstor: + - size: 4m # approximate size limit of single blobovnicza instance, total size will be: size*width^(depth+1), bytes + perm: 0644 # permissions for blobstor files(directories: +x for current user and group) depth: 1 # max depth of object tree storage in key-value DB width: 4 # max width of object tree storage in key-value DB opened_cache_capacity: 50 # maximum number of opened database files + - perm: 0644 # permissions for blobstor files(directories: +x for current user and group) + depth: 5 # max depth of object tree storage in FS gc: remover_batch_size: 200 # number of objects to be removed by the garbage collector @@ -156,12 +156,16 @@ storage: max_batch_size: 100 max_batch_delay: 10ms + compress: true # turn on/off zstd(level 3) compression of stored objects + compression_exclude_content_types: + - audio/* + - video/* + blobstor: - path: tmp/0/blob # blobstor path - compress: true # turn on/off zstd(level 3) compression of stored objects - compression_exclude_content_types: - - audio/* - - video/* + - type: blobovnicza + path: tmp/0/blob/blobovnicza + - type: fstree + path: tmp/0/blob # blobstor path pilorama: path: tmp/0/blob/pilorama.db # path to the pilorama database. If omitted, `pilorama.db` file is created blobstor.path @@ -181,8 +185,10 @@ storage: path: tmp/1/meta # metabase path blobstor: - path: tmp/1/blob # blobstor path - + - type: blobovnicza + path: tmp/1/blob/blobovnicza + - type: fstree + path: tmp/1/blob # blobstor path pilorama: path: tmp/1/blob/pilorama.db diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/blobovnicza.go b/pkg/local_object_storage/blobstor/blobovniczatree/blobovnicza.go index f5fb68c5e..af96894ff 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/blobovnicza.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/blobovnicza.go @@ -11,6 +11,7 @@ import ( "github.com/nspcc-dev/hrw" "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobovnicza" "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor/common" + "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor/compression" apistatus "github.com/nspcc-dev/neofs-sdk-go/client/status" objectSDK "github.com/nspcc-dev/neofs-sdk-go/object" oid "github.com/nspcc-dev/neofs-sdk-go/object/id" @@ -76,8 +77,6 @@ type Blobovniczas struct { // list of active (opened, non-filled) Blobovniczas activeMtx sync.RWMutex active map[string]blobovniczaWithIndex - - onClose []func() } type blobovniczaWithIndex struct { @@ -887,3 +886,8 @@ func u64FromHexString(str string) uint64 { func (b *Blobovniczas) Type() string { return "blobovniczas" } + +// SetCompressor implements common.Storage. +func (b *Blobovniczas) SetCompressor(cc *compression.CConfig) { + b.CConfig = cc +} diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/control.go b/pkg/local_object_storage/blobstor/blobovniczatree/control.go index 58450d9a0..fd772fefc 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/control.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/control.go @@ -19,16 +19,6 @@ func (b *Blobovniczas) Open(readOnly bool) error { func (b *Blobovniczas) Init() error { b.log.Debug("initializing Blobovnicza's") - err := b.CConfig.Init() - if err != nil { - return err - } - b.onClose = append(b.onClose, func() { - if err := b.CConfig.Close(); err != nil { - b.log.Debug("can't close zstd compressor", zap.String("err", err.Error())) - } - }) - if b.readOnly { b.log.Debug("read-only mode, skip blobovniczas initialization...") return nil @@ -78,9 +68,5 @@ func (b *Blobovniczas) Close() error { b.activeMtx.Unlock() - for i := range b.onClose { - b.onClose[i]() - } - return nil } diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/option.go b/pkg/local_object_storage/blobstor/blobovniczatree/option.go index 8f578ce32..b16243b18 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/option.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/option.go @@ -36,7 +36,6 @@ func initConfig(c *cfg) { openedCacheSize: defaultOpenedCacheSize, blzShallowDepth: defaultBlzShallowDepth, blzShallowWidth: defaultBlzShallowWidth, - CConfig: new(compression.CConfig), } } @@ -53,12 +52,6 @@ func WithPermissions(perm fs.FileMode) Option { } } -func WithCompressionConfig(cc *compression.CConfig) Option { - return func(c *cfg) { - c.CConfig = cc - } -} - func WithBlobovniczaShallowWidth(width uint64) Option { return func(c *cfg) { c.blzShallowWidth = width diff --git a/pkg/local_object_storage/blobstor/blobstor.go b/pkg/local_object_storage/blobstor/blobstor.go index 1bd217465..dabc4f18e 100644 --- a/pkg/local_object_storage/blobstor/blobstor.go +++ b/pkg/local_object_storage/blobstor/blobstor.go @@ -1,12 +1,8 @@ package blobstor import ( - "encoding/hex" - "io/fs" - "path/filepath" "sync" - "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor/blobovniczatree" "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor/common" "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor/compression" "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor/fstree" @@ -28,8 +24,6 @@ type BlobStor struct { modeMtx sync.RWMutex mode mode.Mode - - storage [2]SubStorage } type Info = fstree.Info @@ -39,37 +33,12 @@ type Option func(*cfg) type cfg struct { compression.CConfig - - fsTreeDepth int - fsTreeInfo fstree.Info - - smallSizeLimit uint64 - - log *logger.Logger - - blzOpts []blobovniczatree.Option + log *logger.Logger + storage []SubStorage } -const ( - defaultShallowDepth = 4 - defaultPerm = 0700 - - defaultSmallSizeLimit = 1 << 20 // 1MB -) - -const blobovniczaDir = "blobovnicza" - func initConfig(c *cfg) { - *c = cfg{ - fsTreeDepth: defaultShallowDepth, - fsTreeInfo: Info{ - Permissions: defaultPerm, - RootPath: "./", - }, - smallSizeLimit: defaultSmallSizeLimit, - log: zap.L(), - } - c.blzOpts = []blobovniczatree.Option{blobovniczatree.WithCompressionConfig(&c.CConfig)} + c.log = zap.L() } // New creates, initializes and returns new BlobStor instance. @@ -81,23 +50,10 @@ func New(opts ...Option) *BlobStor { opts[i](&bs.cfg) } - bs.storage[0].Storage = blobovniczatree.NewBlobovniczaTree(bs.blzOpts...) - bs.storage[0].Policy = func(_ *objectSDK.Object, data []byte) bool { - return uint64(len(data)) <= bs.cfg.smallSizeLimit + for i := range bs.storage { + bs.storage[i].Storage.SetCompressor(&bs.CConfig) } - bs.storage[1].Storage = &fstree.FSTree{ - Info: bs.cfg.fsTreeInfo, - Depth: bs.cfg.fsTreeDepth, - DirNameLen: hex.EncodedLen(fstree.DirNameLen), - CConfig: &bs.cfg.CConfig, - } - bs.storage[1].Policy = func(*objectSDK.Object, []byte) bool { - return true - } - - bs.blzOpts = nil - return bs } @@ -106,17 +62,17 @@ func (b *BlobStor) SetLogger(l *zap.Logger) { b.log = l } -// WithShallowDepth returns option to set the -// depth of the object file subdirectory tree. -// -// Depth is reduced to maximum value in case of overflow. -func WithShallowDepth(depth int) Option { +// WithStorages provides sub-blobstors. +func WithStorages(st []SubStorage) Option { return func(c *cfg) { - if depth > fstree.MaxDepth { - depth = fstree.MaxDepth - } + c.storage = st + } +} - c.fsTreeDepth = depth +// WithLogger returns option to specify BlobStor's logger. +func WithLogger(l *logger.Logger) Option { + return func(c *cfg) { + c.log = l.With(zap.String("component", "BlobStor")) } } @@ -141,70 +97,3 @@ func WithUncompressableContentTypes(values []string) Option { c.UncompressableContentTypes = values } } - -// WithRootPath returns option to set path to root directory -// of the fs tree to write the objects. -func WithRootPath(rootDir string) Option { - return func(c *cfg) { - c.fsTreeInfo.RootPath = rootDir - c.blzOpts = append(c.blzOpts, blobovniczatree.WithRootPath(filepath.Join(rootDir, blobovniczaDir))) - } -} - -// WithRootPerm returns option to set permission -// bits of the fs tree. -func WithRootPerm(perm fs.FileMode) Option { - return func(c *cfg) { - c.fsTreeInfo.Permissions = perm - c.blzOpts = append(c.blzOpts, blobovniczatree.WithPermissions(perm)) - } -} - -// WithSmallSizeLimit returns option to set maximum size of -// "small" object. -func WithSmallSizeLimit(lim uint64) Option { - return func(c *cfg) { - c.smallSizeLimit = lim - c.blzOpts = append(c.blzOpts, blobovniczatree.WithObjectSizeLimit(lim)) - } -} - -// WithLogger returns option to specify BlobStor's logger. -func WithLogger(l *logger.Logger) Option { - return func(c *cfg) { - c.log = l.With(zap.String("component", "BlobStor")) - c.blzOpts = append(c.blzOpts, blobovniczatree.WithLogger(c.log)) - } -} - -// WithBlobovniczaShallowDepth returns option to specify -// depth of blobovnicza directories. -func WithBlobovniczaShallowDepth(d uint64) Option { - return func(c *cfg) { - c.blzOpts = append(c.blzOpts, blobovniczatree.WithBlobovniczaShallowDepth(d)) - } -} - -// WithBlobovniczaShallowWidth returns option to specify -// width of blobovnicza directories. -func WithBlobovniczaShallowWidth(w uint64) Option { - return func(c *cfg) { - c.blzOpts = append(c.blzOpts, blobovniczatree.WithBlobovniczaShallowWidth(w)) - } -} - -// WithBlobovniczaOpenedCacheSize return option to specify -// maximum number of opened non-active blobovnicza's. -func WithBlobovniczaOpenedCacheSize(sz int) Option { - return func(c *cfg) { - c.blzOpts = append(c.blzOpts, blobovniczatree.WithOpenedCacheSize(sz)) - } -} - -// WithBlobovniczaSize returns option to specify maximum volume -// of each blobovnicza. -func WithBlobovniczaSize(sz uint64) Option { - return func(c *cfg) { - c.blzOpts = append(c.blzOpts, blobovniczatree.WithBlobovniczaSize(sz)) - } -} diff --git a/pkg/local_object_storage/blobstor/blobstor_test.go b/pkg/local_object_storage/blobstor/blobstor_test.go index 459563afe..436fdfe5a 100644 --- a/pkg/local_object_storage/blobstor/blobstor_test.go +++ b/pkg/local_object_storage/blobstor/blobstor_test.go @@ -2,14 +2,35 @@ package blobstor import ( "os" + "path/filepath" "testing" "github.com/nspcc-dev/neofs-node/pkg/core/object" + "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor/blobovniczatree" "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor/common" + "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor/fstree" objectSDK "github.com/nspcc-dev/neofs-sdk-go/object" "github.com/stretchr/testify/require" ) +const blobovniczaDir = "blobovniczas" + +func defaultStorages(p string, smallSizeLimit uint64) []SubStorage { + return []SubStorage{ + { + Storage: blobovniczatree.NewBlobovniczaTree( + blobovniczatree.WithRootPath(filepath.Join(p, "blobovniczas")), + blobovniczatree.WithBlobovniczaShallowWidth(1)), // default width is 16, slow init + Policy: func(_ *objectSDK.Object, data []byte) bool { + return uint64(len(data)) <= smallSizeLimit + }, + }, + { + Storage: fstree.New(fstree.WithPath(p)), + }, + } +} + func TestCompression(t *testing.T) { dir, err := os.MkdirTemp("", "neofs*") require.NoError(t, err) @@ -21,10 +42,9 @@ func TestCompression(t *testing.T) { ) newBlobStor := func(t *testing.T, compress bool) *BlobStor { - bs := New(WithCompressObjects(compress), - WithRootPath(dir), - WithSmallSizeLimit(smallSizeLimit), - WithBlobovniczaShallowWidth(1)) // default width is 16, slow init + bs := New( + WithCompressObjects(compress), + WithStorages(defaultStorages(dir, smallSizeLimit))) require.NoError(t, bs.Open(false)) require.NoError(t, bs.Init()) return bs @@ -86,11 +106,22 @@ func TestBlobstor_needsCompression(t *testing.T) { require.NoError(t, err) t.Cleanup(func() { _ = os.RemoveAll(dir) }) - bs := New(WithCompressObjects(compress), - WithRootPath(dir), - WithSmallSizeLimit(smallSizeLimit), - WithBlobovniczaShallowWidth(1), - WithUncompressableContentTypes(ct)) + bs := New( + WithCompressObjects(compress), + WithUncompressableContentTypes(ct), + WithStorages([]SubStorage{ + { + Storage: blobovniczatree.NewBlobovniczaTree( + blobovniczatree.WithRootPath(filepath.Join(dir, "blobovnicza")), + blobovniczatree.WithBlobovniczaShallowWidth(1)), // default width is 16, slow init + Policy: func(_ *objectSDK.Object, data []byte) bool { + return uint64(len(data)) < smallSizeLimit + }, + }, + { + Storage: fstree.New(fstree.WithPath(dir)), + }, + })) require.NoError(t, bs.Open(false)) require.NoError(t, bs.Init()) return bs diff --git a/pkg/local_object_storage/blobstor/common/storage.go b/pkg/local_object_storage/blobstor/common/storage.go index 13e0641a3..741a863e7 100644 --- a/pkg/local_object_storage/blobstor/common/storage.go +++ b/pkg/local_object_storage/blobstor/common/storage.go @@ -1,5 +1,7 @@ package common +import "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor/compression" + // Storage represents key-value object storage. // It is used as a building block for a blobstor of a shard. type Storage interface { @@ -8,6 +10,7 @@ type Storage interface { Close() error Type() string + SetCompressor(cc *compression.CConfig) Get(GetPrm) (GetRes, error) GetRange(GetRangePrm) (GetRangeRes, error) diff --git a/pkg/local_object_storage/blobstor/control.go b/pkg/local_object_storage/blobstor/control.go index 785f6d7ba..e009080d4 100644 --- a/pkg/local_object_storage/blobstor/control.go +++ b/pkg/local_object_storage/blobstor/control.go @@ -31,6 +31,10 @@ var ErrInitBlobovniczas = errors.New("failure on blobovnicza initialization stag func (b *BlobStor) Init() error { b.log.Debug("initializing...") + if err := b.CConfig.Init(); err != nil { + return err + } + for i := range b.storage { err := b.storage[i].Storage.Init() if err != nil { @@ -55,5 +59,10 @@ func (b *BlobStor) Close() error { continue } } + + err := b.CConfig.Close() + if firstErr == nil { + firstErr = err + } return firstErr } diff --git a/pkg/local_object_storage/blobstor/delete.go b/pkg/local_object_storage/blobstor/delete.go index c605bb550..faa58933b 100644 --- a/pkg/local_object_storage/blobstor/delete.go +++ b/pkg/local_object_storage/blobstor/delete.go @@ -26,7 +26,7 @@ func (b *BlobStor) Delete(prm common.DeletePrm) (common.DeleteRes, error) { } } if len(prm.StorageID) == 0 { - return b.storage[1].Storage.Delete(prm) + return b.storage[len(b.storage)-1].Storage.Delete(prm) } return b.storage[0].Storage.Delete(prm) } diff --git a/pkg/local_object_storage/blobstor/exists_test.go b/pkg/local_object_storage/blobstor/exists_test.go index 7a1ad1872..1db0c2cd9 100644 --- a/pkg/local_object_storage/blobstor/exists_test.go +++ b/pkg/local_object_storage/blobstor/exists_test.go @@ -20,9 +20,8 @@ func TestExists(t *testing.T) { const smallSizeLimit = 512 - b := New(WithRootPath(dir), - WithSmallSizeLimit(smallSizeLimit), - WithBlobovniczaShallowWidth(1)) // default width is 16, slow init + b := New( + WithStorages(defaultStorages(dir, smallSizeLimit))) require.NoError(t, b.Open(false)) require.NoError(t, b.Init()) @@ -65,7 +64,7 @@ func TestExists(t *testing.T) { require.NotEmpty(t, bigDir) require.NoError(t, os.Chmod(dir, 0)) - t.Cleanup(func() { require.NoError(t, os.Chmod(dir, b.fsTreeInfo.Permissions)) }) + t.Cleanup(func() { require.NoError(t, os.Chmod(dir, 0777)) }) // Object exists, first error is logged. prm.Address = objectCore.AddressOf(objects[0]) diff --git a/pkg/local_object_storage/blobstor/fstree/control.go b/pkg/local_object_storage/blobstor/fstree/control.go index eb82cf179..93d0d08e4 100644 --- a/pkg/local_object_storage/blobstor/fstree/control.go +++ b/pkg/local_object_storage/blobstor/fstree/control.go @@ -1,10 +1,16 @@ package fstree +import ( + "github.com/nspcc-dev/neofs-node/pkg/util" +) + // Open implements common.Storage. func (*FSTree) Open(bool) error { return nil } // Init implements common.Storage. -func (*FSTree) Init() error { return nil } +func (t *FSTree) Init() error { + return util.MkdirAllX(t.RootPath, t.Permissions) +} // Close implements common.Storage. func (*FSTree) Close() error { return nil } diff --git a/pkg/local_object_storage/blobstor/fstree/fstree.go b/pkg/local_object_storage/blobstor/fstree/fstree.go index f987857bb..d430dfcfd 100644 --- a/pkg/local_object_storage/blobstor/fstree/fstree.go +++ b/pkg/local_object_storage/blobstor/fstree/fstree.go @@ -45,6 +45,23 @@ const ( var _ common.Storage = (*FSTree)(nil) +func New(opts ...Option) *FSTree { + f := &FSTree{ + Info: Info{ + Permissions: 0700, + RootPath: "./", + }, + CConfig: nil, + Depth: 4, + DirNameLen: DirNameLen, + } + for i := range opts { + opts[i](f) + } + + return f +} + func stringifyAddress(addr oid.Address) string { return addr.Object().EncodeToString() + "." + addr.Container().EncodeToString() } @@ -299,3 +316,8 @@ func (t *FSTree) NumberOfObjects() (uint64, error) { func (*FSTree) Type() string { return "fstree" } + +// SetCompressor implements common.Storage. +func (t *FSTree) SetCompressor(cc *compression.CConfig) { + t.CConfig = cc +} diff --git a/pkg/local_object_storage/blobstor/fstree/option.go b/pkg/local_object_storage/blobstor/fstree/option.go new file mode 100644 index 000000000..fd3cc272c --- /dev/null +++ b/pkg/local_object_storage/blobstor/fstree/option.go @@ -0,0 +1,31 @@ +package fstree + +import ( + "io/fs" +) + +type Option func(*FSTree) + +func WithDepth(d int) Option { + return func(f *FSTree) { + f.Depth = d + } +} + +func WithDirNameLen(l int) Option { + return func(f *FSTree) { + f.DirNameLen = l + } +} + +func WithPerm(p fs.FileMode) Option { + return func(f *FSTree) { + f.Permissions = p + } +} + +func WithPath(p string) Option { + return func(f *FSTree) { + f.RootPath = p + } +} diff --git a/pkg/local_object_storage/blobstor/get.go b/pkg/local_object_storage/blobstor/get.go index 25bfd7b8e..da90999a9 100644 --- a/pkg/local_object_storage/blobstor/get.go +++ b/pkg/local_object_storage/blobstor/get.go @@ -23,7 +23,7 @@ func (b *BlobStor) Get(prm common.GetPrm) (common.GetRes, error) { return common.GetRes{}, errNotFound } if len(prm.StorageID) == 0 { - return b.storage[1].Storage.Get(prm) + return b.storage[len(b.storage)-1].Storage.Get(prm) } return b.storage[0].Storage.Get(prm) } diff --git a/pkg/local_object_storage/blobstor/get_range.go b/pkg/local_object_storage/blobstor/get_range.go index c5289f871..722a667f4 100644 --- a/pkg/local_object_storage/blobstor/get_range.go +++ b/pkg/local_object_storage/blobstor/get_range.go @@ -23,7 +23,7 @@ func (b *BlobStor) GetRange(prm common.GetRangePrm) (common.GetRangeRes, error) return common.GetRangeRes{}, errNotFound } if len(prm.StorageID) == 0 { - return b.storage[1].Storage.GetRange(prm) + return b.storage[len(b.storage)-1].Storage.GetRange(prm) } return b.storage[0].Storage.GetRange(prm) } diff --git a/pkg/local_object_storage/blobstor/info.go b/pkg/local_object_storage/blobstor/info.go index 87ce5606b..60a143ce1 100644 --- a/pkg/local_object_storage/blobstor/info.go +++ b/pkg/local_object_storage/blobstor/info.go @@ -4,5 +4,10 @@ import "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor/fstree // DumpInfo returns information about blob stor. func (b *BlobStor) DumpInfo() fstree.Info { - return b.cfg.fsTreeInfo + for i := range b.storage { + if b.storage[i].Storage.Type() == "fstree" { + return b.storage[i].Storage.(*fstree.FSTree).Info + } + } + return fstree.Info{} } diff --git a/pkg/local_object_storage/blobstor/iterate.go b/pkg/local_object_storage/blobstor/iterate.go index 23f67b923..81cd31d0b 100644 --- a/pkg/local_object_storage/blobstor/iterate.go +++ b/pkg/local_object_storage/blobstor/iterate.go @@ -19,7 +19,7 @@ func (b *BlobStor) Iterate(prm common.IteratePrm) (common.IterateRes, error) { for i := range b.storage { _, err := b.storage[i].Storage.Iterate(prm) if err != nil && !prm.IgnoreErrors { - return common.IterateRes{}, fmt.Errorf("blobovnizas iterator failure: %w", err) + return common.IterateRes{}, fmt.Errorf("blobstor iterator failure: %w", err) } } return common.IterateRes{}, nil diff --git a/pkg/local_object_storage/blobstor/iterate_test.go b/pkg/local_object_storage/blobstor/iterate_test.go index f5dfd5f33..78191e924 100644 --- a/pkg/local_object_storage/blobstor/iterate_test.go +++ b/pkg/local_object_storage/blobstor/iterate_test.go @@ -18,11 +18,8 @@ func TestIterateObjects(t *testing.T) { // create BlobStor instance blobStor := New( + WithStorages(defaultStorages(p, smalSz)), WithCompressObjects(true), - WithRootPath(p), - WithSmallSizeLimit(smalSz), - WithBlobovniczaShallowWidth(1), - WithBlobovniczaShallowDepth(1), ) defer os.RemoveAll(p) diff --git a/pkg/local_object_storage/blobstor/put.go b/pkg/local_object_storage/blobstor/put.go index 9e7b7c86d..6e0f45539 100644 --- a/pkg/local_object_storage/blobstor/put.go +++ b/pkg/local_object_storage/blobstor/put.go @@ -33,7 +33,7 @@ func (b *BlobStor) Put(prm common.PutPrm) (common.PutRes, error) { } for i := range b.storage { - if b.storage[i].Policy(prm.Object, prm.RawData) { + if b.storage[i].Policy == nil || b.storage[i].Policy(prm.Object, prm.RawData) { res, err := b.storage[i].Storage.Put(prm) if err == nil { storagelog.Write(b.log, diff --git a/pkg/local_object_storage/engine/engine_test.go b/pkg/local_object_storage/engine/engine_test.go index c4ebc84e0..94fbc4457 100644 --- a/pkg/local_object_storage/engine/engine_test.go +++ b/pkg/local_object_storage/engine/engine_test.go @@ -8,6 +8,8 @@ import ( "testing" "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor" + "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor/blobovniczatree" + "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor/fstree" meta "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/metabase" "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/pilorama" "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/shard" @@ -94,6 +96,26 @@ func testNewEngineWithShards(shards ...*shard.Shard) *StorageEngine { return engine } +func newStorages(root string, smallSize uint64) []blobstor.SubStorage { + return []blobstor.SubStorage{ + { + Storage: blobovniczatree.NewBlobovniczaTree( + blobovniczatree.WithRootPath(filepath.Join(root, "blobovnicza")), + blobovniczatree.WithBlobovniczaShallowDepth(1), + blobovniczatree.WithBlobovniczaShallowWidth(1), + blobovniczatree.WithPermissions(0700)), + Policy: func(_ *object.Object, data []byte) bool { + return uint64(len(data)) < smallSize + }, + }, + { + Storage: fstree.New( + fstree.WithPath(root), + fstree.WithDepth(1)), + }, + } +} + func testNewShard(t testing.TB, id int) *shard.Shard { sid, err := generateShardID() require.NoError(t, err) @@ -102,11 +124,9 @@ func testNewShard(t testing.TB, id int) *shard.Shard { shard.WithID(sid), shard.WithLogger(zap.L()), shard.WithBlobStorOptions( - blobstor.WithRootPath(filepath.Join(t.Name(), fmt.Sprintf("%d.blobstor", id))), - blobstor.WithBlobovniczaShallowWidth(2), - blobstor.WithBlobovniczaShallowDepth(2), - blobstor.WithRootPerm(0700), - ), + blobstor.WithStorages( + newStorages(filepath.Join(t.Name(), fmt.Sprintf("%d.blobstor", id)), + 1<<20))), shard.WithPiloramaOptions(pilorama.WithPath(filepath.Join(t.Name(), fmt.Sprintf("%d.pilorama", id)))), shard.WithMetaBaseOptions( meta.WithPath(filepath.Join(t.Name(), fmt.Sprintf("%d.metabase", id))), @@ -125,10 +145,9 @@ func testEngineFromShardOpts(t *testing.T, num int, extraOpts func(int) []shard. for i := 0; i < num; i++ { _, err := engine.AddShard(append([]shard.Option{ shard.WithBlobStorOptions( - blobstor.WithRootPath(filepath.Join(t.Name(), fmt.Sprintf("blobstor%d", i))), - blobstor.WithBlobovniczaShallowWidth(1), - blobstor.WithBlobovniczaShallowDepth(1), - blobstor.WithRootPerm(0700), + blobstor.WithStorages( + newStorages(filepath.Join(t.Name(), fmt.Sprintf("blobstor%d", i)), + 1<<20)), ), shard.WithMetaBaseOptions( meta.WithPath(filepath.Join(t.Name(), fmt.Sprintf("metabase%d", i))), diff --git a/pkg/local_object_storage/engine/error_test.go b/pkg/local_object_storage/engine/error_test.go index 3c56a461a..2439d667b 100644 --- a/pkg/local_object_storage/engine/error_test.go +++ b/pkg/local_object_storage/engine/error_test.go @@ -43,12 +43,7 @@ func newEngineWithErrorThreshold(t testing.TB, dir string, errThreshold uint32) ids[i], err = e.AddShard( shard.WithLogger(zaptest.NewLogger(t)), shard.WithBlobStorOptions( - blobstor.WithRootPath(filepath.Join(dir, strconv.Itoa(i))), - blobstor.WithShallowDepth(1), - blobstor.WithBlobovniczaShallowWidth(1), - blobstor.WithBlobovniczaShallowDepth(1), - blobstor.WithSmallSizeLimit(errSmallSize), - blobstor.WithRootPerm(0700)), + blobstor.WithStorages(newStorages(filepath.Join(dir, strconv.Itoa(i)), errSmallSize))), shard.WithMetaBaseOptions( meta.WithPath(filepath.Join(dir, fmt.Sprintf("%d.metabase", i))), meta.WithPermissions(0700), diff --git a/pkg/local_object_storage/shard/control_test.go b/pkg/local_object_storage/shard/control_test.go index 21bb4d91c..cfb26716f 100644 --- a/pkg/local_object_storage/shard/control_test.go +++ b/pkg/local_object_storage/shard/control_test.go @@ -39,11 +39,14 @@ func TestShardOpen(t *testing.T) { return New( WithLogger(zaptest.NewLogger(t)), WithBlobStorOptions( - blobstor.WithRootPath(filepath.Join(dir, "blob")), - blobstor.WithShallowDepth(1), - blobstor.WithSmallSizeLimit(1), - blobstor.WithBlobovniczaShallowWidth(1), - blobstor.WithBlobovniczaShallowDepth(1)), + blobstor.WithStorages([]blobstor.SubStorage{ + { + Storage: fstree.New( + fstree.WithDirNameLen(2), + fstree.WithPath(filepath.Join(dir, "blob")), + fstree.WithDepth(1)), + }, + })), WithMetaBaseOptions(meta.WithPath(metaPath), meta.WithEpochState(epochState{})), WithPiloramaOptions( pilorama.WithPath(filepath.Join(dir, "pilorama"))), @@ -81,11 +84,15 @@ func TestRefillMetabaseCorrupted(t *testing.T) { dir := t.TempDir() blobOpts := []blobstor.Option{ - blobstor.WithRootPath(filepath.Join(dir, "blob")), - blobstor.WithShallowDepth(1), - blobstor.WithSmallSizeLimit(1), - blobstor.WithBlobovniczaShallowWidth(1), - blobstor.WithBlobovniczaShallowDepth(1)} + blobstor.WithStorages([]blobstor.SubStorage{ + { + Storage: fstree.New( + fstree.WithDirNameLen(2), + fstree.WithPath(filepath.Join(dir, "blob")), + fstree.WithDepth(1)), + }, + }), + } sh := New( WithBlobStorOptions(blobOpts...), @@ -134,9 +141,13 @@ func TestRefillMetabase(t *testing.T) { defer os.RemoveAll(p) blobOpts := []blobstor.Option{ - blobstor.WithRootPath(filepath.Join(p, "blob")), - blobstor.WithBlobovniczaShallowWidth(1), - blobstor.WithBlobovniczaShallowDepth(1), + blobstor.WithStorages([]blobstor.SubStorage{ + { + Storage: fstree.New( + fstree.WithPath(filepath.Join(p, "blob")), + fstree.WithDepth(1)), + }, + }), } sh := New( diff --git a/pkg/local_object_storage/shard/dump_test.go b/pkg/local_object_storage/shard/dump_test.go index 5f2ac1582..3cf0004e1 100644 --- a/pkg/local_object_storage/shard/dump_test.go +++ b/pkg/local_object_storage/shard/dump_test.go @@ -13,6 +13,8 @@ import ( "github.com/nspcc-dev/neofs-node/pkg/core/object" "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobovnicza" "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor" + "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor/blobovniczatree" + "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor/fstree" "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/shard" "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/shard/mode" "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/writecache" @@ -52,9 +54,7 @@ func testDump(t *testing.T, objCount int, hasWriteCache bool) { writecache.WithMaxObjectSize(wcBigObjectSize), writecache.WithLogger(zaptest.NewLogger(t)), }, - []blobstor.Option{ - blobstor.WithLogger(zaptest.NewLogger(t)), - }) + nil) } defer releaseShard(sh, t) @@ -186,7 +186,7 @@ func testDump(t *testing.T, objCount int, hasWriteCache bool) { t.Run("skip errors", func(t *testing.T) { sh := newCustomShard(t, filepath.Join(t.TempDir(), "ignore"), false, nil, nil) - defer releaseShard(sh, t) + t.Cleanup(func() { require.NoError(t, sh.Close()) }) var restorePrm shard.RestorePrm restorePrm.WithPath(out) @@ -292,14 +292,27 @@ func TestDumpIgnoreErrors(t *testing.T) { dir := t.TempDir() bsPath := filepath.Join(dir, "blob") - bsOpts := []blobstor.Option{ - blobstor.WithSmallSizeLimit(bsSmallObjectSize), - blobstor.WithRootPath(bsPath), - blobstor.WithCompressObjects(true), - blobstor.WithShallowDepth(1), - blobstor.WithBlobovniczaShallowDepth(1), - blobstor.WithBlobovniczaShallowWidth(2), - blobstor.WithBlobovniczaOpenedCacheSize(1), + bsOpts := func(sw uint64) []blobstor.Option { + return []blobstor.Option{ + blobstor.WithCompressObjects(true), + blobstor.WithStorages([]blobstor.SubStorage{ + { + Storage: blobovniczatree.NewBlobovniczaTree( + blobovniczatree.WithRootPath(filepath.Join(bsPath, "blobovnicza")), + blobovniczatree.WithBlobovniczaShallowDepth(1), + blobovniczatree.WithBlobovniczaShallowWidth(sw), + blobovniczatree.WithOpenedCacheSize(1)), + Policy: func(_ *objectSDK.Object, data []byte) bool { + return len(data) < bsSmallObjectSize + }, + }, + { + Storage: fstree.New( + fstree.WithPath(bsPath), + fstree.WithDepth(1)), + }, + }), + } } wcPath := filepath.Join(dir, "writecache") wcOpts := []writecache.Option{ @@ -307,7 +320,7 @@ func TestDumpIgnoreErrors(t *testing.T) { writecache.WithSmallObjectSize(wcSmallObjectSize), writecache.WithMaxObjectSize(wcBigObjectSize), } - sh := newCustomShard(t, dir, true, wcOpts, bsOpts) + sh := newCustomShard(t, dir, true, wcOpts, bsOpts(2)) objects := make([]*objectSDK.Object, objCount) for i := 0; i < objCount; i++ { @@ -355,8 +368,7 @@ func TestDumpIgnoreErrors(t *testing.T) { require.NoError(t, os.MkdirAll(filepath.Join(bsPath, "ZZ"), 0)) } - bsOpts = append(bsOpts, blobstor.WithBlobovniczaShallowWidth(3)) - sh = newCustomShard(t, dir, true, wcOpts, bsOpts) + sh = newCustomShard(t, dir, true, wcOpts, bsOpts(3)) require.NoError(t, sh.SetMode(mode.ReadOnly)) { diff --git a/pkg/local_object_storage/shard/lock_test.go b/pkg/local_object_storage/shard/lock_test.go index 1abc6c087..a4b55276b 100644 --- a/pkg/local_object_storage/shard/lock_test.go +++ b/pkg/local_object_storage/shard/lock_test.go @@ -7,6 +7,8 @@ import ( objectcore "github.com/nspcc-dev/neofs-node/pkg/core/object" "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor" + "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor/blobovniczatree" + "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor/fstree" meta "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/metabase" "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/shard" apistatus "github.com/nspcc-dev/neofs-sdk-go/client/status" @@ -24,11 +26,21 @@ func TestShard_Lock(t *testing.T) { opts := []shard.Option{ shard.WithLogger(zap.NewNop()), shard.WithBlobStorOptions( - []blobstor.Option{ - blobstor.WithRootPath(filepath.Join(rootPath, "blob")), - blobstor.WithBlobovniczaShallowWidth(2), - blobstor.WithBlobovniczaShallowDepth(2), - }..., + blobstor.WithStorages([]blobstor.SubStorage{ + { + Storage: blobovniczatree.NewBlobovniczaTree( + blobovniczatree.WithRootPath(filepath.Join(rootPath, "blob", "blobovnicza")), + blobovniczatree.WithBlobovniczaShallowDepth(2), + blobovniczatree.WithBlobovniczaShallowWidth(2)), + Policy: func(_ *object.Object, data []byte) bool { + return len(data) <= 1<<20 + }, + }, + { + Storage: fstree.New( + fstree.WithPath(filepath.Join(rootPath, "blob"))), + }, + }), ), shard.WithMetaBaseOptions( meta.WithPath(filepath.Join(rootPath, "meta")), diff --git a/pkg/local_object_storage/shard/range_test.go b/pkg/local_object_storage/shard/range_test.go index 10769cfd7..f9a66172a 100644 --- a/pkg/local_object_storage/shard/range_test.go +++ b/pkg/local_object_storage/shard/range_test.go @@ -2,16 +2,20 @@ package shard_test import ( "math" + "path/filepath" "testing" "github.com/nspcc-dev/neo-go/pkg/util/slice" "github.com/nspcc-dev/neofs-node/pkg/core/object" "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor" + "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor/blobovniczatree" + "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor/fstree" "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/shard" "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/writecache" apistatus "github.com/nspcc-dev/neofs-sdk-go/client/status" objectSDK "github.com/nspcc-dev/neofs-sdk-go/object" "github.com/stretchr/testify/require" + "go.uber.org/zap/zaptest" ) func TestShard_GetRange(t *testing.T) { @@ -62,7 +66,22 @@ func testShardGetRange(t *testing.T, hasWriteCache bool) { sh := newCustomShard(t, t.TempDir(), hasWriteCache, []writecache.Option{writecache.WithMaxMemSize(0), writecache.WithMaxObjectSize(writeCacheMaxSize)}, - []blobstor.Option{blobstor.WithSmallSizeLimit(smallObjectSize)}) + []blobstor.Option{blobstor.WithStorages([]blobstor.SubStorage{ + { + Storage: blobovniczatree.NewBlobovniczaTree( + blobovniczatree.WithLogger(zaptest.NewLogger(t)), + blobovniczatree.WithRootPath(filepath.Join(t.TempDir(), "blob", "blobovnicza")), + blobovniczatree.WithBlobovniczaShallowDepth(1), + blobovniczatree.WithBlobovniczaShallowWidth(1)), + Policy: func(_ *objectSDK.Object, data []byte) bool { + return len(data) <= smallObjectSize + }, + }, + { + Storage: fstree.New( + fstree.WithPath(filepath.Join(t.TempDir(), "blob"))), + }, + })}) defer releaseShard(sh, t) for _, tc := range testCases { diff --git a/pkg/local_object_storage/shard/shard_test.go b/pkg/local_object_storage/shard/shard_test.go index ed783a394..a4c91e49b 100644 --- a/pkg/local_object_storage/shard/shard_test.go +++ b/pkg/local_object_storage/shard/shard_test.go @@ -4,12 +4,12 @@ import ( "crypto/sha256" "math" "math/rand" - "os" "path/filepath" - "strings" "testing" "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor" + "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor/blobovniczatree" + "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor/fstree" meta "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/metabase" "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/pilorama" "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/shard" @@ -24,6 +24,7 @@ import ( "github.com/nspcc-dev/tzhash/tz" "github.com/stretchr/testify/require" "go.uber.org/zap" + "go.uber.org/zap/zaptest" ) type epochState struct{} @@ -45,15 +46,31 @@ func newCustomShard(t testing.TB, rootPath string, enableWriteCache bool, wcOpts rootPath = filepath.Join(rootPath, "nowc") } + if bsOpts == nil { + bsOpts = []blobstor.Option{ + blobstor.WithLogger(zaptest.NewLogger(t)), + blobstor.WithStorages([]blobstor.SubStorage{ + { + Storage: blobovniczatree.NewBlobovniczaTree( + blobovniczatree.WithLogger(zaptest.NewLogger(t)), + blobovniczatree.WithRootPath(filepath.Join(rootPath, "blob", "blobovnicza")), + blobovniczatree.WithBlobovniczaShallowDepth(1), + blobovniczatree.WithBlobovniczaShallowWidth(1)), + Policy: func(_ *object.Object, data []byte) bool { + return len(data) <= 1<<20 + }, + }, + { + Storage: fstree.New( + fstree.WithPath(filepath.Join(rootPath, "blob"))), + }, + }), + } + } + opts := []shard.Option{ shard.WithLogger(zap.L()), - shard.WithBlobStorOptions( - append([]blobstor.Option{ - blobstor.WithRootPath(filepath.Join(rootPath, "blob")), - blobstor.WithBlobovniczaShallowWidth(2), - blobstor.WithBlobovniczaShallowDepth(2), - }, bsOpts...)..., - ), + shard.WithBlobStorOptions(bsOpts...), shard.WithMetaBaseOptions( meta.WithPath(filepath.Join(rootPath, "meta")), meta.WithEpochState(epochState{}), @@ -76,8 +93,7 @@ func newCustomShard(t testing.TB, rootPath string, enableWriteCache bool, wcOpts } func releaseShard(s *shard.Shard, t testing.TB) { - s.Close() - os.RemoveAll(strings.Split(t.Name(), string(os.PathSeparator))[0]) + require.NoError(t, s.Close()) } func generateObject(t *testing.T) *object.Object { diff --git a/pkg/local_object_storage/writecache/mode.go b/pkg/local_object_storage/writecache/mode.go index 9f6d8bb4f..389567252 100644 --- a/pkg/local_object_storage/writecache/mode.go +++ b/pkg/local_object_storage/writecache/mode.go @@ -27,7 +27,6 @@ func (c *cache) SetMode(m mode.Mode) error { if err := c.db.Close(); err != nil { return fmt.Errorf("can't close write-cache database: %w", err) } - c.db = nil } // Suspend producers to ensure there are channel send operations in fly.