TrueCloudLab/frostfs-node
20
1
Fork 28
Code Issues 107 Pull requests 4 Projects Releases 2 Packages 4 Activity Actions

[#1523] neofs-node: Refactor configuration

Browse source 
1. Move compression parameters to the `shard` section.
2. Allow to use multiple sub-storage components in the blobstor.

Signed-off-by: Evgenii Stratonikov <evgeniy@nspcc.ru>
This commit is contained in:
Evgenii Stratonikov 2022-07-11 15:34:17 +03:00 • committed by fyrchik
parent 13cdbde2e2
commit 26b4a258e0
37 changed files with 595 additions and 419 deletions
Download patch file Download diff file Expand all files Collapse all files

139
pkg/local_object_storage/blobstor/blobstor.go
View file

@ -1,12 +1,8 @@
package blobstor
import (
"encoding/hex"
"io/fs"
"path/filepath"
"sync"
"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor/blobovniczatree"
"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor/common"
"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor/compression"
"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor/fstree"
@ -28,8 +24,6 @@ type BlobStor struct {
modeMtx sync.RWMutex
mode mode.Mode
storage [2]SubStorage
}
type Info = fstree.Info
@ -39,37 +33,12 @@ type Option func(*cfg)
type cfg struct {
compression.CConfig
fsTreeDepth int
fsTreeInfo fstree.Info
smallSizeLimit uint64
log *logger.Logger
blzOpts []blobovniczatree.Option
log *logger.Logger
storage []SubStorage
}
const (
defaultShallowDepth = 4
defaultPerm = 0700
defaultSmallSizeLimit = 1 << 20 // 1MB
)
const blobovniczaDir = "blobovnicza"
func initConfig(c *cfg) {
*c = cfg{
fsTreeDepth: defaultShallowDepth,
fsTreeInfo: Info{
Permissions: defaultPerm,
RootPath: "./",
},
smallSizeLimit: defaultSmallSizeLimit,
log: zap.L(),
}
c.blzOpts = []blobovniczatree.Option{blobovniczatree.WithCompressionConfig(&c.CConfig)}
c.log = zap.L()
}
// New creates, initializes and returns new BlobStor instance.
@ -81,23 +50,10 @@ func New(opts ...Option) *BlobStor {
opts[i](&bs.cfg)
}
bs.storage[0].Storage = blobovniczatree.NewBlobovniczaTree(bs.blzOpts...)
bs.storage[0].Policy = func(_ *objectSDK.Object, data []byte) bool {
return uint64(len(data)) <= bs.cfg.smallSizeLimit
for i := range bs.storage {
bs.storage[i].Storage.SetCompressor(&bs.CConfig)
}
bs.storage[1].Storage = &fstree.FSTree{
Info: bs.cfg.fsTreeInfo,
Depth: bs.cfg.fsTreeDepth,
DirNameLen: hex.EncodedLen(fstree.DirNameLen),
CConfig: &bs.cfg.CConfig,
}
bs.storage[1].Policy = func(*objectSDK.Object, []byte) bool {
return true
}
bs.blzOpts = nil
return bs
}
@ -106,17 +62,17 @@ func (b *BlobStor) SetLogger(l *zap.Logger) {
b.log = l
}
// WithShallowDepth returns option to set the
// depth of the object file subdirectory tree.
//
// Depth is reduced to maximum value in case of overflow.
func WithShallowDepth(depth int) Option {
// WithStorages provides sub-blobstors.
func WithStorages(st []SubStorage) Option {
return func(c *cfg) {
if depth > fstree.MaxDepth {
depth = fstree.MaxDepth
}
c.storage = st
}
}
c.fsTreeDepth = depth
// WithLogger returns option to specify BlobStor's logger.
func WithLogger(l *logger.Logger) Option {
return func(c *cfg) {
c.log = l.With(zap.String("component", "BlobStor"))
}
}
@ -141,70 +97,3 @@ func WithUncompressableContentTypes(values []string) Option {
c.UncompressableContentTypes = values
}
}
// WithRootPath returns option to set path to root directory
// of the fs tree to write the objects.
func WithRootPath(rootDir string) Option {
return func(c *cfg) {
c.fsTreeInfo.RootPath = rootDir
c.blzOpts = append(c.blzOpts, blobovniczatree.WithRootPath(filepath.Join(rootDir, blobovniczaDir)))
}
}
// WithRootPerm returns option to set permission
// bits of the fs tree.
func WithRootPerm(perm fs.FileMode) Option {
return func(c *cfg) {
c.fsTreeInfo.Permissions = perm
c.blzOpts = append(c.blzOpts, blobovniczatree.WithPermissions(perm))
}
}
// WithSmallSizeLimit returns option to set maximum size of
// "small" object.
func WithSmallSizeLimit(lim uint64) Option {
return func(c *cfg) {
c.smallSizeLimit = lim
c.blzOpts = append(c.blzOpts, blobovniczatree.WithObjectSizeLimit(lim))
}
}
// WithLogger returns option to specify BlobStor's logger.
func WithLogger(l *logger.Logger) Option {
return func(c *cfg) {
c.log = l.With(zap.String("component", "BlobStor"))
c.blzOpts = append(c.blzOpts, blobovniczatree.WithLogger(c.log))
}
}
// WithBlobovniczaShallowDepth returns option to specify
// depth of blobovnicza directories.
func WithBlobovniczaShallowDepth(d uint64) Option {
return func(c *cfg) {
c.blzOpts = append(c.blzOpts, blobovniczatree.WithBlobovniczaShallowDepth(d))
}
}
// WithBlobovniczaShallowWidth returns option to specify
// width of blobovnicza directories.
func WithBlobovniczaShallowWidth(w uint64) Option {
return func(c *cfg) {
c.blzOpts = append(c.blzOpts, blobovniczatree.WithBlobovniczaShallowWidth(w))
}
}
// WithBlobovniczaOpenedCacheSize return option to specify
// maximum number of opened non-active blobovnicza's.
func WithBlobovniczaOpenedCacheSize(sz int) Option {
return func(c *cfg) {
c.blzOpts = append(c.blzOpts, blobovniczatree.WithOpenedCacheSize(sz))
}
}
// WithBlobovniczaSize returns option to specify maximum volume
// of each blobovnicza.
func WithBlobovniczaSize(sz uint64) Option {
return func(c *cfg) {
c.blzOpts = append(c.blzOpts, blobovniczatree.WithBlobovniczaSize(sz))
}
}