TrueCloudLab/frostfs-node
20
1
Fork 27
Code Issues 101 Pull requests 5 Projects Releases 1 Activity Actions

[#1505] pilorama: Allow to customize database parameters

Browse source 
Signed-off-by: Evgenii Stratonikov <evgeniy@nspcc.ru>
This commit is contained in:
Evgenii Stratonikov 2022-06-09 11:09:18 +03:00 committed by fyrchik
parent f80e52fbea
commit 26041f18bf
15 changed files with 253 additions and 20 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

15
cmd/neofs-node/config.go
View file

@ -24,6 +24,7 @@ import (
"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor"
"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/engine"
meta "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/metabase"
"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/pilorama"
"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/shard"
"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/writecache"
"github.com/nspcc-dev/neofs-node/pkg/metrics"
@ -421,6 +422,19 @@ func initShardOptions(c *cfg) {
metabaseCfg := sc.Metabase()
gcCfg := sc.GC()
piloramaCfg := sc.Pilorama()
piloramaPath := piloramaCfg.Path()
if piloramaPath == "" {
piloramaPath = filepath.Join(blobStorCfg.Path(), "pilorama.db")
}
piloramaOpts := []pilorama.Option{
pilorama.WithPath(piloramaPath),
pilorama.WithPerm(piloramaCfg.Perm()),
pilorama.WithNoSync(piloramaCfg.NoSync()),
pilorama.WithMaxBatchSize(piloramaCfg.MaxBatchSize()),
pilorama.WithMaxBatchDelay(piloramaCfg.MaxBatchDelay())}
metaPath := metabaseCfg.Path()
metaPerm := metabaseCfg.BoltDB().Perm()
fatalOnErr(util.MkdirAllX(filepath.Dir(metaPath), metaPerm))
@ -456,6 +470,7 @@ func initShardOptions(c *cfg) {
Timeout: 100 * time.Millisecond,
}),
),
shard.WithPiloramaOptions(piloramaOpts...),
shard.WithWriteCache(writeCacheCfg.Enabled()),
shard.WithWriteCacheOptions(writeCacheOpts...),
shard.WithRemoverBatchSize(gcCfg.RemoverBatchSize()),

14
cmd/neofs-node/config/engine/config_test.go
View file

@ -8,6 +8,7 @@ import (
"github.com/nspcc-dev/neofs-node/cmd/neofs-node/config"
engineconfig "github.com/nspcc-dev/neofs-node/cmd/neofs-node/config/engine"
shardconfig "github.com/nspcc-dev/neofs-node/cmd/neofs-node/config/engine/shard"
piloramaconfig "github.com/nspcc-dev/neofs-node/cmd/neofs-node/config/engine/shard/pilorama"
configtest "github.com/nspcc-dev/neofs-node/cmd/neofs-node/config/test"
"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/shard"
"github.com/stretchr/testify/require"
@ -53,10 +54,17 @@ func TestEngineSection(t *testing.T) {
meta := sc.Metabase()
blob := sc.BlobStor()
blz := blob.Blobovnicza()
pl := sc.Pilorama()
gc := sc.GC()
switch num {
case 0:
require.Equal(t, "tmp/0/blob/pilorama.db", pl.Path())
require.Equal(t, fs.FileMode(piloramaconfig.PermDefault), pl.Perm())
require.False(t, pl.NoSync())
require.Equal(t, pl.MaxBatchDelay(), 10*time.Millisecond)
require.Equal(t, pl.MaxBatchSize(), 200)
require.Equal(t, false, wc.Enabled())
require.Equal(t, "tmp/0/cache", wc.Path())
@ -89,6 +97,12 @@ func TestEngineSection(t *testing.T) {
require.Equal(t, false, sc.RefillMetabase())
require.Equal(t, shard.ModeReadOnly, sc.Mode())
case 1:
require.Equal(t, "tmp/1/blob/pilorama.db", pl.Path())
require.Equal(t, fs.FileMode(0644), pl.Perm())
require.True(t, pl.NoSync())
require.Equal(t, 5*time.Millisecond, pl.MaxBatchDelay())
require.Equal(t, 100, pl.MaxBatchSize())
require.Equal(t, true, wc.Enabled())
require.Equal(t, "tmp/1/cache", wc.Path())

9
cmd/neofs-node/config/engine/shard/config.go
View file

@ -7,6 +7,7 @@ import (
blobstorconfig "github.com/nspcc-dev/neofs-node/cmd/neofs-node/config/engine/shard/blobstor"
gcconfig "github.com/nspcc-dev/neofs-node/cmd/neofs-node/config/engine/shard/gc"
metabaseconfig "github.com/nspcc-dev/neofs-node/cmd/neofs-node/config/engine/shard/metabase"
piloramaconfig "github.com/nspcc-dev/neofs-node/cmd/neofs-node/config/engine/shard/pilorama"
writecacheconfig "github.com/nspcc-dev/neofs-node/cmd/neofs-node/config/engine/shard/writecache"
"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/shard"
)
@ -44,6 +45,14 @@ func (x *Config) WriteCache() *writecacheconfig.Config {
)
}
// Pilorama returns "pilorama" subsection as a piloramaconfig.Config.
func (x *Config) Pilorama() *piloramaconfig.Config {
return piloramaconfig.From(
(*config.Config)(x).
Sub("pilorama"),
)
}
// GC returns "gc" subsection as a gcconfig.Config.
func (x *Config) GC() *gcconfig.Config {
return gcconfig.From(

70
cmd/neofs-node/config/engine/shard/pilorama/config.go Normal file
View file

@ -0,0 +1,70 @@
package piloramaconfig
import (
"io/fs"
"time"
"github.com/nspcc-dev/neofs-node/cmd/neofs-node/config"
)
// Config is a wrapper over the config section
// which provides access to Metabase configurations.
type Config config.Config
const (
// PermDefault is a default permission bits for metabase file.
PermDefault = 0660
)
// From wraps config section into Config.
func From(c *config.Config) *Config {
return (*Config)(c)
}
// Path returns the value of "path" config parameter.
//
// Returns empty string if missing, for compatibility with older configurations.
func (x *Config) Path() string {
return config.String((*config.Config)(x), "path")
}
// Perm returns the value of "perm" config parameter as a fs.FileMode.
//
// Returns PermDefault if the value is not a positive number.
func (x *Config) Perm() fs.FileMode {
p := config.UintSafe((*config.Config)(x), "perm")
if p == 0 {
p = PermDefault
}
return fs.FileMode(p)
}
// NoSync returns the value of "no_sync" config parameter as a bool value.
//
// Returns false if the value is not a boolean.
func (x *Config) NoSync() bool {
return config.BoolSafe((*config.Config)(x), "no_sync")
}
// MaxBatchDelay returns the value of "max_batch_delay" config parameter.
//
// Returns 0 if the value is not a positive number.
func (x *Config) MaxBatchDelay() time.Duration {
d := config.DurationSafe((*config.Config)(x), "max_batch_delay")
if d <= 0 {
d = 0
}
return d
}
// MaxBatchSize returns the value of "max_batch_size" config parameter.
//
// Returns 0 if the value is not a positive number.
func (x *Config) MaxBatchSize() int {
s := int(config.IntSafe((*config.Config)(x), "max_batch_size"))
if s <= 0 {
s = 0
}
return s
}

10
config/example/node.env
View file

@ -107,6 +107,10 @@ NEOFS_STORAGE_SHARD_0_BLOBSTOR_BLOBOVNICZA_SIZE=4194304
NEOFS_STORAGE_SHARD_0_BLOBSTOR_BLOBOVNICZA_DEPTH=1
NEOFS_STORAGE_SHARD_0_BLOBSTOR_BLOBOVNICZA_WIDTH=4
NEOFS_STORAGE_SHARD_0_BLOBSTOR_BLOBOVNICZA_OPENED_CACHE_CAPACITY=50
### Pilorama config
NEOFS_STORAGE_SHARD_0_PILORAMA_PATH="tmp/0/blob/pilorama.db"
NEOFS_STORAGE_SHARD_0_PILORAMA_MAX_BATCH_DELAY=10ms
NEOFS_STORAGE_SHARD_0_PILORAMA_MAX_BATCH_SIZE=200
### GC config
#### Limit of the single data remover's batching operation in number of objects
NEOFS_STORAGE_SHARD_0_GC_REMOVER_BATCH_SIZE=150
@ -142,6 +146,12 @@ NEOFS_STORAGE_SHARD_1_BLOBSTOR_BLOBOVNICZA_SIZE=4194304
NEOFS_STORAGE_SHARD_1_BLOBSTOR_BLOBOVNICZA_DEPTH=1
NEOFS_STORAGE_SHARD_1_BLOBSTOR_BLOBOVNICZA_WIDTH=4
NEOFS_STORAGE_SHARD_1_BLOBSTOR_BLOBOVNICZA_OPENED_CACHE_CAPACITY=50
### Pilorama config
NEOFS_STORAGE_SHARD_1_PILORAMA_PATH="tmp/1/blob/pilorama.db"
NEOFS_STORAGE_SHARD_1_PILORAMA_PERM=0644
NEOFS_STORAGE_SHARD_1_PILORAMA_NO_SYNC=true
NEOFS_STORAGE_SHARD_1_PILORAMA_MAX_BATCH_DELAY=5ms
NEOFS_STORAGE_SHARD_1_PILORAMA_MAX_BATCH_SIZE=100
### GC config
#### Limit of the single data remover's batching operation in number of objects
NEOFS_STORAGE_SHARD_1_GC_REMOVER_BATCH_SIZE=200

12
config/example/node.json
View file

@ -158,6 +158,11 @@
"opened_cache_capacity": 50
}
},
"pilorama": {
"path": "tmp/0/blob/pilorama.db",
"max_batch_delay": "10ms",
"max_batch_size": 200
},
"gc": {
"remover_batch_size": 150,
"remover_sleep_interval": "2m"
@ -194,6 +199,13 @@
"opened_cache_capacity": 50
}
},
"pilorama": {
"path": "tmp/1/blob/pilorama.db",
"perm": "0644",
"no_sync": true,
"max_batch_delay": "5ms",
"max_batch_size": 100
},
"gc": {
"remover_batch_size": 200,
"remover_sleep_interval": "5m"

15
config/example/node.yaml
View file

@ -120,6 +120,10 @@ storage:
max_batch_size: 200
max_batch_delay: 20ms
pilorama:
max_batch_delay: 5ms # maximum delay for a batch of operations to be executed
max_batch_size: 100 # maximum amount of operations in a single batch
blobstor:
compress: false # turn on/off zstd(level 3) compression of stored objects
perm: 0644 # permissions for blobstor files(directories: +x for current user and group)
@ -157,6 +161,11 @@ storage:
- audio/*
- video/*
pilorama:
path: tmp/0/blob/pilorama.db # path to the pilorama database. If omitted, `pilorama.db` file is created blobstor.path
max_batch_delay: 10ms
max_batch_size: 200
gc:
remover_batch_size: 150 # number of objects to be removed by the garbage collector
remover_sleep_interval: 2m # frequency of the garbage collector invocation
@ -171,3 +180,9 @@ storage:
blobstor:
path: tmp/1/blob # blobstor path
pilorama:
path: tmp/1/blob/pilorama.db
no_sync: true # USE WITH CAUTION. Return to user before pages have been persisted.
perm: 0644 # permission to use for the database file and intermediate directories

7
pkg/local_object_storage/engine/engine_test.go
View file

@ -8,6 +8,7 @@ import (
"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor"
meta "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/metabase"
"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/pilorama"
"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/shard"
"github.com/nspcc-dev/neofs-sdk-go/checksum"
checksumtest "github.com/nspcc-dev/neofs-sdk-go/checksum/test"
@ -99,6 +100,7 @@ func testNewShard(t testing.TB, id int) *shard.Shard {
blobstor.WithBlobovniczaShallowDepth(2),
blobstor.WithRootPerm(0700),
),
shard.WithPiloramaOptions(pilorama.WithPath(filepath.Join(t.Name(), fmt.Sprintf("%d.pilorama", id)))),
shard.WithMetaBaseOptions(
meta.WithPath(filepath.Join(t.Name(), fmt.Sprintf("%d.metabase", id))),
meta.WithPermissions(0700),
@ -123,7 +125,10 @@ func testEngineFromShardOpts(t *testing.T, num int, extraOpts func(int) []shard.
shard.WithMetaBaseOptions(
meta.WithPath(filepath.Join(t.Name(), fmt.Sprintf("metabase%d", i))),
meta.WithPermissions(0700),
)}, extraOpts(i)...)...)
),
shard.WithPiloramaOptions(
pilorama.WithPath(filepath.Join(t.Name(), fmt.Sprintf("pilorama%d", i)))),
}, extraOpts(i)...)...)
require.NoError(t, err)
}

6
pkg/local_object_storage/engine/error_test.go
View file

@ -10,6 +10,7 @@ import (
"github.com/nspcc-dev/neofs-node/pkg/core/object"
"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor"
meta "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/metabase"
"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/pilorama"
"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/shard"
apistatus "github.com/nspcc-dev/neofs-sdk-go/client/status"
cidtest "github.com/nspcc-dev/neofs-sdk-go/container/id/test"
@ -49,7 +50,10 @@ func newEngineWithErrorThreshold(t testing.TB, dir string, errThreshold uint32)
blobstor.WithRootPerm(0700)),
shard.WithMetaBaseOptions(
meta.WithPath(filepath.Join(dir, fmt.Sprintf("%d.metabase", i))),
meta.WithPermissions(0700)))
meta.WithPermissions(0700)),
shard.WithPiloramaOptions(
pilorama.WithPath(filepath.Join(dir, fmt.Sprintf("%d.pilorama", i))),
pilorama.WithPerm(0700)))
require.NoError(t, err)
}
require.NoError(t, e.Open())

46
pkg/local_object_storage/pilorama/boltdb.go
View file

@ -3,22 +3,22 @@ package pilorama
import (
"bytes"
"encoding/binary"
"fmt"
"math/rand"
"os"
"path/filepath"
"github.com/nspcc-dev/neo-go/pkg/io"
"github.com/nspcc-dev/neofs-node/pkg/util"
cidSDK "github.com/nspcc-dev/neofs-sdk-go/container/id"
"go.etcd.io/bbolt"
)
type boltForest struct {
path string
db *bbolt.DB
db *bbolt.DB
cfg
}
const defaultMaxBatchSize = 10
var (
dataBucket = []byte{0}
logBucket = []byte{1}
@ -41,25 +41,41 @@ var (
// 'm' + node (id) -> serialized meta
// 'c' + parent (id) + child (id) -> 0/1
// 'i' + 0 + attrKey + 0 + attrValue + 0 + parent (id) + node (id) -> 0/1 (1 for automatically created nodes)
func NewBoltForest(path string) ForestStorage {
return &boltForest{path: path}
func NewBoltForest(opts ...Option) ForestStorage {
b := boltForest{
cfg: cfg{
perm: os.ModePerm,
maxBatchDelay: bbolt.DefaultMaxBatchDelay,
maxBatchSize: bbolt.DefaultMaxBatchSize,
},
}
for i := range opts {
opts[i](&b.cfg)
}
return &b
}
func (t *boltForest) Init() error { return nil }
func (t *boltForest) Open() error {
if err := os.MkdirAll(filepath.Dir(t.path), os.ModePerm); err != nil {
return err
}
db, err := bbolt.Open(t.path, os.ModePerm, bbolt.DefaultOptions)
err := util.MkdirAllX(filepath.Dir(t.path), t.perm)
if err != nil {
return err
return fmt.Errorf("can't create dir %s for the pilorama: %w", t.path, err)
}
db.MaxBatchSize = defaultMaxBatchSize
t.db = db
opts := *bbolt.DefaultOptions
opts.NoSync = t.noSync
return db.Update(func(tx *bbolt.Tx) error {
t.db, err = bbolt.Open(t.path, t.perm, &opts)
if err != nil {
return fmt.Errorf("can't open the pilorama DB: %w", err)
}
t.db.MaxBatchSize = t.maxBatchSize
t.db.MaxBatchDelay = t.maxBatchDelay
return t.db.Update(func(tx *bbolt.Tx) error {
_, err := tx.CreateBucketIfNotExists(dataBucket)
if err != nil {
return err

2
pkg/local_object_storage/pilorama/forest_test.go
View file

@ -31,7 +31,7 @@ var providers = []struct {
tmpDir, err := os.MkdirTemp(os.TempDir(), "*")
require.NoError(t, err)
f := NewBoltForest(filepath.Join(tmpDir, "test.db"))
f := NewBoltForest(WithPath(filepath.Join(tmpDir, "test.db")))
require.NoError(t, f.Init())
require.NoError(t, f.Open())
t.Cleanup(func() {

46
pkg/local_object_storage/pilorama/option.go Normal file
View file

@ -0,0 +1,46 @@
package pilorama
import (
"io/fs"
"time"
)
type Option func(*cfg)
type cfg struct {
path string
perm fs.FileMode
noSync bool
maxBatchDelay time.Duration
maxBatchSize int
}
func WithPath(path string) Option {
return func(c *cfg) {
c.path = path
}
}
func WithPerm(perm fs.FileMode) Option {
return func(c *cfg) {
c.perm = perm
}
}
func WithNoSync(noSync bool) Option {
return func(c *cfg) {
c.noSync = noSync
}
}
func WithMaxBatchDelay(d time.Duration) Option {
return func(c *cfg) {
c.maxBatchDelay = d
}
}
func WithMaxBatchSize(size int) Option {
return func(c *cfg) {
c.maxBatchSize = size
}
}

7
pkg/local_object_storage/shard/control_test.go
View file

@ -9,6 +9,7 @@ import (
"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor"
"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor/fstree"
meta "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/metabase"
"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/pilorama"
apistatus "github.com/nspcc-dev/neofs-sdk-go/client/status"
cid "github.com/nspcc-dev/neofs-sdk-go/container/id"
cidtest "github.com/nspcc-dev/neofs-sdk-go/container/id/test"
@ -31,6 +32,7 @@ func TestRefillMetabaseCorrupted(t *testing.T) {
sh := New(
WithBlobStorOptions(blobOpts...),
WithPiloramaOptions(pilorama.WithPath(filepath.Join(dir, "pilorama"))),
WithMetaBaseOptions(meta.WithPath(filepath.Join(dir, "meta"))))
require.NoError(t, sh.Open())
require.NoError(t, sh.Init())
@ -55,6 +57,7 @@ func TestRefillMetabaseCorrupted(t *testing.T) {
sh = New(
WithBlobStorOptions(blobOpts...),
WithPiloramaOptions(pilorama.WithPath(filepath.Join(dir, "pilorama"))),
WithMetaBaseOptions(meta.WithPath(filepath.Join(dir, "meta_new"))),
WithRefillMetabase(true))
require.NoError(t, sh.Open())
@ -83,6 +86,8 @@ func TestRefillMetabase(t *testing.T) {
WithMetaBaseOptions(
meta.WithPath(filepath.Join(p, "meta")),
),
WithPiloramaOptions(
pilorama.WithPath(filepath.Join(p, "pilorama"))),
)
// open Blobstor
@ -246,6 +251,8 @@ func TestRefillMetabase(t *testing.T) {
WithMetaBaseOptions(
meta.WithPath(filepath.Join(p, "meta_restored")),
),
WithPiloramaOptions(
pilorama.WithPath(filepath.Join(p, "pilorama_another"))),
)
// open Blobstor

12
pkg/local_object_storage/shard/shard.go
View file

@ -2,7 +2,6 @@ package shard
import (
"context"
"path/filepath"
"sync"
"time"
@ -62,6 +61,8 @@ type cfg struct {
writeCacheOpts []writecache.Option
piloramaOpts []pilorama.Option
log *logger.Logger
gcCfg *gcCfg
@ -108,7 +109,7 @@ func New(opts ...Option) *Shard {
metaBase: mb,
writeCache: writeCache,
tsSource: c.tsSource,
pilorama: pilorama.NewBoltForest(filepath.Join(bs.DumpInfo().RootPath, "pilorama.db")),
pilorama: pilorama.NewBoltForest(c.piloramaOpts...),
}
s.fillInfo()
@ -144,6 +145,13 @@ func WithWriteCacheOptions(opts ...writecache.Option) Option {
}
}
// WithPiloramaOptions returns option to set internal write cache options.
func WithPiloramaOptions(opts ...pilorama.Option) Option {
return func(c *cfg) {
c.piloramaOpts = opts
}
}
// WithLogger returns option to set Shard's logger.
func WithLogger(l *logger.Logger) Option {
return func(c *cfg) {

2
pkg/local_object_storage/shard/shard_test.go
View file

@ -10,6 +10,7 @@ import (
"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor"
meta "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/metabase"
"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/pilorama"
"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/shard"
"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/writecache"
"github.com/nspcc-dev/neofs-sdk-go/checksum"
@ -49,6 +50,7 @@ func newCustomShard(t testing.TB, rootPath string, enableWriteCache bool, wcOpts
shard.WithMetaBaseOptions(
meta.WithPath(filepath.Join(rootPath, "meta")),
),
shard.WithPiloramaOptions(pilorama.WithPath(filepath.Join(rootPath, "pilorama"))),
shard.WithWriteCache(enableWriteCache),
shard.WithWriteCacheOptions(
append(