ale64bit/frostfs-node
1
0
Fork 0
forked from TrueCloudLab/frostfs-node
Code Issues Pull requests Projects Releases Packages Wiki Activity

[#1505] pilorama: Allow to customize database parameters

Browse source 
Signed-off-by: Evgenii Stratonikov <evgeniy@nspcc.ru>
This commit is contained in:
Evgenii Stratonikov 2022-06-09 11:09:18 +03:00
parent 2c8a87a469
commit 1fed255c5b
15 changed files with 253 additions and 18 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

15
cmd/neofs-node/config.go
View file

@ -24,6 +24,7 @@ import (
"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor"
"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/engine"
meta "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/metabase"
"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/pilorama"
"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/shard"
"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/writecache"
"github.com/nspcc-dev/neofs-node/pkg/metrics"
@ -421,6 +422,19 @@ func initShardOptions(c *cfg) {
metabaseCfg := sc.Metabase()
gcCfg := sc.GC()
piloramaCfg := sc.Pilorama()
piloramaPath := piloramaCfg.Path()
if piloramaPath == "" {
piloramaPath = filepath.Join(blobStorCfg.Path(), "pilorama.db")
}
piloramaOpts := []pilorama.Option{
pilorama.WithPath(piloramaPath),
pilorama.WithPerm(piloramaCfg.Perm()),
pilorama.WithNoSync(piloramaCfg.NoSync()),
pilorama.WithMaxBatchSize(piloramaCfg.MaxBatchSize()),
pilorama.WithMaxBatchDelay(piloramaCfg.MaxBatchDelay())}
metaPath := metabaseCfg.Path()
metaPerm := metabaseCfg.BoltDB().Perm()
fatalOnErr(util.MkdirAllX(filepath.Dir(metaPath), metaPerm))
@ -456,6 +470,7 @@ func initShardOptions(c *cfg) {
Timeout: 100 * time.Millisecond,
}),
),
shard.WithPiloramaOptions(piloramaOpts...),
shard.WithWriteCache(writeCacheCfg.Enabled()),
shard.WithWriteCacheOptions(writeCacheOpts...),
shard.WithRemoverBatchSize(gcCfg.RemoverBatchSize()),

14
cmd/neofs-node/config/engine/config_test.go
View file

@ -8,6 +8,7 @@ import (
"github.com/nspcc-dev/neofs-node/cmd/neofs-node/config"
engineconfig "github.com/nspcc-dev/neofs-node/cmd/neofs-node/config/engine"
shardconfig "github.com/nspcc-dev/neofs-node/cmd/neofs-node/config/engine/shard"
piloramaconfig "github.com/nspcc-dev/neofs-node/cmd/neofs-node/config/engine/shard/pilorama"
configtest "github.com/nspcc-dev/neofs-node/cmd/neofs-node/config/test"
"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/shard"
"github.com/stretchr/testify/require"
@ -53,10 +54,17 @@ func TestEngineSection(t *testing.T) {
meta := sc.Metabase()
blob := sc.BlobStor()
blz := blob.Blobovnicza()
pl := sc.Pilorama()
gc := sc.GC()
switch num {
case 0:
require.Equal(t, "tmp/0/blob/pilorama.db", pl.Path())
require.Equal(t, fs.FileMode(piloramaconfig.PermDefault), pl.Perm())
require.False(t, pl.NoSync())
require.Equal(t, pl.MaxBatchDelay(), 10*time.Millisecond)
require.Equal(t, pl.MaxBatchSize(), 200)
require.Equal(t, false, wc.Enabled())
require.Equal(t, "tmp/0/cache", wc.Path())
@ -89,6 +97,12 @@ func TestEngineSection(t *testing.T) {
require.Equal(t, false, sc.RefillMetabase())
require.Equal(t, shard.ModeReadOnly, sc.Mode())
case 1:
require.Equal(t, "tmp/1/blob/pilorama.db", pl.Path())
require.Equal(t, fs.FileMode(0644), pl.Perm())
require.True(t, pl.NoSync())
require.Equal(t, 5*time.Millisecond, pl.MaxBatchDelay())
require.Equal(t, 100, pl.MaxBatchSize())
require.Equal(t, true, wc.Enabled())
require.Equal(t, "tmp/1/cache", wc.Path())

9
cmd/neofs-node/config/engine/shard/config.go
View file

@ -7,6 +7,7 @@ import (
blobstorconfig "github.com/nspcc-dev/neofs-node/cmd/neofs-node/config/engine/shard/blobstor"
gcconfig "github.com/nspcc-dev/neofs-node/cmd/neofs-node/config/engine/shard/gc"
metabaseconfig "github.com/nspcc-dev/neofs-node/cmd/neofs-node/config/engine/shard/metabase"
piloramaconfig "github.com/nspcc-dev/neofs-node/cmd/neofs-node/config/engine/shard/pilorama"
writecacheconfig "github.com/nspcc-dev/neofs-node/cmd/neofs-node/config/engine/shard/writecache"
"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/shard"
)
@ -44,6 +45,14 @@ func (x *Config) WriteCache() *writecacheconfig.Config {
)
}
// Pilorama returns "pilorama" subsection as a piloramaconfig.Config.
func (x *Config) Pilorama() *piloramaconfig.Config {
return piloramaconfig.From(
(*config.Config)(x).
Sub("pilorama"),
)
}
// GC returns "gc" subsection as a gcconfig.Config.
func (x *Config) GC() *gcconfig.Config {
return gcconfig.From(

70
cmd/neofs-node/config/engine/shard/pilorama/config.go Normal file
View file

@ -0,0 +1,70 @@
package piloramaconfig
import (
"io/fs"
"time"
"github.com/nspcc-dev/neofs-node/cmd/neofs-node/config"
)
// Config is a wrapper over the config section
// which provides access to Metabase configurations.
type Config config.Config
const (
// PermDefault is a default permission bits for metabase file.
PermDefault = 0660
)
// From wraps config section into Config.
func From(c *config.Config) *Config {
return (*Config)(c)
}
// Path returns the value of "path" config parameter.
//
// Returns empty string if missing, for compatibility with older configurations.
func (x *Config) Path() string {
return config.String((*config.Config)(x), "path")
}
// Perm returns the value of "perm" config parameter as a fs.FileMode.
//
// Returns PermDefault if the value is not a positive number.
func (x *Config) Perm() fs.FileMode {
p := config.UintSafe((*config.Config)(x), "perm")
if p == 0 {
p = PermDefault
}
return fs.FileMode(p)
}
// NoSync returns the value of "no_sync" config parameter as a bool value.
//
// Returns false if the value is not a boolean.
func (x *Config) NoSync() bool {
return config.BoolSafe((*config.Config)(x), "no_sync")
}
// MaxBatchDelay returns the value of "max_batch_delay" config parameter.
//
// Returns 0 if the value is not a positive number.
func (x *Config) MaxBatchDelay() time.Duration {
d := config.DurationSafe((*config.Config)(x), "max_batch_delay")
if d <= 0 {
d = 0
}
return d
}
// MaxBatchSize returns the value of "max_batch_size" config parameter.
//
// Returns 0 if the value is not a positive number.
func (x *Config) MaxBatchSize() int {
s := int(config.IntSafe((*config.Config)(x), "max_batch_size"))
if s <= 0 {
s = 0
}
return s
}

10
config/example/node.env
View file

@ -105,6 +105,10 @@ NEOFS_STORAGE_SHARD_0_BLOBSTOR_BLOBOVNICZA_SIZE=4194304
NEOFS_STORAGE_SHARD_0_BLOBSTOR_BLOBOVNICZA_DEPTH=1
NEOFS_STORAGE_SHARD_0_BLOBSTOR_BLOBOVNICZA_WIDTH=4
NEOFS_STORAGE_SHARD_0_BLOBSTOR_BLOBOVNICZA_OPENED_CACHE_CAPACITY=50
### Pilorama config
NEOFS_STORAGE_SHARD_0_PILORAMA_PATH="tmp/0/blob/pilorama.db"
NEOFS_STORAGE_SHARD_0_PILORAMA_MAX_BATCH_DELAY=10ms
NEOFS_STORAGE_SHARD_0_PILORAMA_MAX_BATCH_SIZE=200
### GC config
#### Limit of the single data remover's batching operation in number of objects
NEOFS_STORAGE_SHARD_0_GC_REMOVER_BATCH_SIZE=150
@ -140,6 +144,12 @@ NEOFS_STORAGE_SHARD_1_BLOBSTOR_BLOBOVNICZA_SIZE=4194304
NEOFS_STORAGE_SHARD_1_BLOBSTOR_BLOBOVNICZA_DEPTH=1
NEOFS_STORAGE_SHARD_1_BLOBSTOR_BLOBOVNICZA_WIDTH=4
NEOFS_STORAGE_SHARD_1_BLOBSTOR_BLOBOVNICZA_OPENED_CACHE_CAPACITY=50
### Pilorama config
NEOFS_STORAGE_SHARD_1_PILORAMA_PATH="tmp/1/blob/pilorama.db"
NEOFS_STORAGE_SHARD_1_PILORAMA_PERM=0644
NEOFS_STORAGE_SHARD_1_PILORAMA_NO_SYNC=true
NEOFS_STORAGE_SHARD_1_PILORAMA_MAX_BATCH_DELAY=5ms
NEOFS_STORAGE_SHARD_1_PILORAMA_MAX_BATCH_SIZE=100
### GC config
#### Limit of the single data remover's batching operation in number of objects
NEOFS_STORAGE_SHARD_1_GC_REMOVER_BATCH_SIZE=200

12
config/example/node.json
View file

@ -156,6 +156,11 @@
"opened_cache_capacity": 50
}
},
"pilorama": {
"path": "tmp/0/blob/pilorama.db",
"max_batch_delay": "10ms",
"max_batch_size": 200
},
"gc": {
"remover_batch_size": 150,
"remover_sleep_interval": "2m"
@ -192,6 +197,13 @@
"opened_cache_capacity": 50
}
},
"pilorama": {
"path": "tmp/1/blob/pilorama.db",
"perm": "0644",
"no_sync": true,
"max_batch_delay": "5ms",
"max_batch_size": 100
},
"gc": {
"remover_batch_size": 200,
"remover_sleep_interval": "5m"

15
config/example/node.yaml
View file

@ -119,6 +119,10 @@ storage:
max_batch_size: 200
max_batch_delay: 20ms
pilorama:
max_batch_delay: 5ms # maximum delay for a batch of operations to be executed
max_batch_size: 100 # maximum amount of operations in a single batch
blobstor:
compress: false # turn on/off zstd(level 3) compression of stored objects
perm: 0644 # permissions for blobstor files(directories: +x for current user and group)
@ -157,6 +161,11 @@ storage:
- audio/*
- video/*
pilorama:
path: tmp/0/blob/pilorama.db # path to the pilorama database. If omitted, `pilorama.db` file is created blobstor.path
max_batch_delay: 10ms
max_batch_size: 200
gc:
remover_batch_size: 150 # number of objects to be removed by the garbage collector
remover_sleep_interval: 2m # frequency of the garbage collector invocation
@ -171,3 +180,9 @@ storage:
blobstor:
path: tmp/1/blob # blobstor path
pilorama:
path: tmp/1/blob/pilorama.db
no_sync: true # USE WITH CAUTION. Return to user before pages have been persisted.
perm: 0644 # permission to use for the database file and intermediate directories

7
pkg/local_object_storage/engine/engine_test.go
View file

@ -8,6 +8,7 @@ import (
"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor"
meta "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/metabase"
"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/pilorama"
"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/shard"
"github.com/nspcc-dev/neofs-sdk-go/checksum"
checksumtest "github.com/nspcc-dev/neofs-sdk-go/checksum/test"
@ -99,6 +100,7 @@ func testNewShard(t testing.TB, id int) *shard.Shard {
blobstor.WithBlobovniczaShallowDepth(2),
blobstor.WithRootPerm(0700),
),
shard.WithPiloramaOptions(pilorama.WithPath(filepath.Join(t.Name(), fmt.Sprintf("%d.pilorama", id)))),
shard.WithMetaBaseOptions(
meta.WithPath(filepath.Join(t.Name(), fmt.Sprintf("%d.metabase", id))),
meta.WithPermissions(0700),
@ -123,7 +125,10 @@ func testEngineFromShardOpts(t *testing.T, num int, extraOpts func(int) []shard.
shard.WithMetaBaseOptions(
meta.WithPath(filepath.Join(t.Name(), fmt.Sprintf("metabase%d", i))),
meta.WithPermissions(0700),
)}, extraOpts(i)...)...)
),
shard.WithPiloramaOptions(
pilorama.WithPath(filepath.Join(t.Name(), fmt.Sprintf("pilorama%d", i)))),
}, extraOpts(i)...)...)
require.NoError(t, err)
}

6
pkg/local_object_storage/engine/error_test.go
View file

@ -10,6 +10,7 @@ import (
"github.com/nspcc-dev/neofs-node/pkg/core/object"
"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor"
meta "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/metabase"
"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/pilorama"
"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/shard"
cidtest "github.com/nspcc-dev/neofs-sdk-go/container/id/test"
objectSDK "github.com/nspcc-dev/neofs-sdk-go/object"
@ -48,7 +49,10 @@ func newEngineWithErrorThreshold(t testing.TB, dir string, errThreshold uint32)
blobstor.WithRootPerm(0700)),
shard.WithMetaBaseOptions(
meta.WithPath(filepath.Join(dir, fmt.Sprintf("%d.metabase", i))),
meta.WithPermissions(0700)))
meta.WithPermissions(0700)),
shard.WithPiloramaOptions(
pilorama.WithPath(filepath.Join(dir, fmt.Sprintf("%d.pilorama", i))),
pilorama.WithPerm(0700)))
require.NoError(t, err)
}
require.NoError(t, e.Open())

42
pkg/local_object_storage/pilorama/boltdb.go
View file

@ -3,18 +3,20 @@ package pilorama
import (
"bytes"
"encoding/binary"
"fmt"
"math/rand"
"os"
"path/filepath"
"github.com/nspcc-dev/neo-go/pkg/io"
"github.com/nspcc-dev/neofs-node/pkg/util"
cidSDK "github.com/nspcc-dev/neofs-sdk-go/container/id"
"go.etcd.io/bbolt"
)
type boltForest struct {
path string
db *bbolt.DB
cfg
}
const defaultMaxBatchSize = 10
@ -41,25 +43,41 @@ var (
// 'm' + node (id) -> serialized meta
// 'c' + parent (id) + child (id) -> 0/1
// 'i' + 0 + attrKey + 0 + attrValue + 0 + parent (id) + node (id) -> 0/1 (1 for automatically created nodes)
func NewBoltForest(path string) ForestStorage {
return &boltForest{path: path}
func NewBoltForest(opts ...Option) ForestStorage {
b := boltForest{
cfg: cfg{
perm: os.ModePerm,
maxBatchDelay: bbolt.DefaultMaxBatchDelay,
maxBatchSize: bbolt.DefaultMaxBatchSize,
},
}
for i := range opts {
opts[i](&b.cfg)
}
return &b
}
func (t *boltForest) Init() error { return nil }
func (t *boltForest) Open() error {
if err := os.MkdirAll(filepath.Dir(t.path), os.ModePerm); err != nil {
return err
}
db, err := bbolt.Open(t.path, os.ModePerm, bbolt.DefaultOptions)
err := util.MkdirAllX(filepath.Dir(t.path), t.perm)
if err != nil {
return err
return fmt.Errorf("can't create dir %s for the pilorama: %w", t.path, err)
}
db.MaxBatchSize = defaultMaxBatchSize
t.db = db
opts := *bbolt.DefaultOptions
opts.NoSync = t.noSync
return db.Update(func(tx *bbolt.Tx) error {
t.db, err = bbolt.Open(t.path, t.perm, &opts)
if err != nil {
return fmt.Errorf("can't open the pilorama DB: %w", err)
}
t.db.MaxBatchSize = t.maxBatchSize
t.db.MaxBatchDelay = t.maxBatchDelay
return t.db.Update(func(tx *bbolt.Tx) error {
_, err := tx.CreateBucketIfNotExists(dataBucket)
if err != nil {
return err

2
pkg/local_object_storage/pilorama/forest_test.go
View file

@ -31,7 +31,7 @@ var providers = []struct {
tmpDir, err := os.MkdirTemp(os.TempDir(), "*")
require.NoError(t, err)
f := NewBoltForest(filepath.Join(tmpDir, "test.db"))
f := NewBoltForest(WithPath(filepath.Join(tmpDir, "test.db")))
require.NoError(t, f.Init())
require.NoError(t, f.Open())
t.Cleanup(func() {

46
pkg/local_object_storage/pilorama/option.go Normal file
View file

@ -0,0 +1,46 @@
package pilorama
import (
"io/fs"
"time"
)
type Option func(*cfg)
type cfg struct {
path string
perm fs.FileMode
noSync bool
maxBatchDelay time.Duration
maxBatchSize int
}
func WithPath(path string) Option {
return func(c *cfg) {
c.path = path
}
}
func WithPerm(perm fs.FileMode) Option {
return func(c *cfg) {
c.perm = perm
}
}
func WithNoSync(noSync bool) Option {
return func(c *cfg) {
c.noSync = noSync
}
}
func WithMaxBatchDelay(d time.Duration) Option {
return func(c *cfg) {
c.maxBatchDelay = d
}
}
func WithMaxBatchSize(size int) Option {
return func(c *cfg) {
c.maxBatchSize = size
}
}

7
pkg/local_object_storage/shard/control_test.go
View file

@ -9,6 +9,7 @@ import (
"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor"
"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor/fstree"
meta "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/metabase"
"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/pilorama"
apistatus "github.com/nspcc-dev/neofs-sdk-go/client/status"
cid "github.com/nspcc-dev/neofs-sdk-go/container/id"
cidtest "github.com/nspcc-dev/neofs-sdk-go/container/id/test"
@ -31,6 +32,7 @@ func TestRefillMetabaseCorrupted(t *testing.T) {
sh := New(
WithBlobStorOptions(blobOpts...),
WithPiloramaOptions(pilorama.WithPath(filepath.Join(dir, "pilorama"))),
WithMetaBaseOptions(meta.WithPath(filepath.Join(dir, "meta"))))
require.NoError(t, sh.Open())
require.NoError(t, sh.Init())
@ -55,6 +57,7 @@ func TestRefillMetabaseCorrupted(t *testing.T) {
sh = New(
WithBlobStorOptions(blobOpts...),
WithPiloramaOptions(pilorama.WithPath(filepath.Join(dir, "pilorama"))),
WithMetaBaseOptions(meta.WithPath(filepath.Join(dir, "meta_new"))),
WithRefillMetabase(true))
require.NoError(t, sh.Open())
@ -83,6 +86,8 @@ func TestRefillMetabase(t *testing.T) {
WithMetaBaseOptions(
meta.WithPath(filepath.Join(p, "meta")),
),
WithPiloramaOptions(
pilorama.WithPath(filepath.Join(p, "pilorama"))),
)
// open Blobstor
@ -246,6 +251,8 @@ func TestRefillMetabase(t *testing.T) {
WithMetaBaseOptions(
meta.WithPath(filepath.Join(p, "meta_restored")),
),
WithPiloramaOptions(
pilorama.WithPath(filepath.Join(p, "pilorama_another"))),
)
// open Blobstor

12
pkg/local_object_storage/shard/shard.go
View file

@ -2,7 +2,6 @@ package shard
import (
"context"
"path/filepath"
"sync"
"time"
@ -59,6 +58,8 @@ type cfg struct {
writeCacheOpts []writecache.Option
piloramaOpts []pilorama.Option
log *logger.Logger
gcCfg *gcCfg
@ -103,7 +104,7 @@ func New(opts ...Option) *Shard {
metaBase: mb,
writeCache: writeCache,
tsSource: c.tsSource,
pilorama: pilorama.NewBoltForest(filepath.Join(bs.DumpInfo().RootPath, "pilorama.db")),
pilorama: pilorama.NewBoltForest(c.piloramaOpts...),
}
s.fillInfo()
@ -139,6 +140,13 @@ func WithWriteCacheOptions(opts ...writecache.Option) Option {
}
}
// WithPiloramaOptions returns option to set internal write cache options.
func WithPiloramaOptions(opts ...pilorama.Option) Option {
return func(c *cfg) {
c.piloramaOpts = opts
}
}
// WithLogger returns option to set Shard's logger.
func WithLogger(l *logger.Logger) Option {
return func(c *cfg) {

2
pkg/local_object_storage/shard/shard_test.go
View file

@ -10,6 +10,7 @@ import (
"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor"
meta "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/metabase"
"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/pilorama"
"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/shard"
"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/writecache"
"github.com/nspcc-dev/neofs-sdk-go/checksum"
@ -49,6 +50,7 @@ func newCustomShard(t testing.TB, rootPath string, enableWriteCache bool, wcOpts
shard.WithMetaBaseOptions(
meta.WithPath(filepath.Join(rootPath, "meta")),
),
shard.WithPiloramaOptions(pilorama.WithPath(filepath.Join(rootPath, "pilorama"))),
shard.WithWriteCache(enableWriteCache),
shard.WithWriteCacheOptions(
append(