[#1505] pilorama: Allow to customize database parameters

Signed-off-by: Evgenii Stratonikov <evgeniy@nspcc.ru>

cmd/neofs-node/config.go

@@ -24,6 +24,7 @@ import (
 	"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor"
 	"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/engine"
 	meta "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/metabase"
+	"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/pilorama"
 	"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/shard"
 	"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/writecache"
 	"github.com/nspcc-dev/neofs-node/pkg/metrics"
@@ -421,6 +422,19 @@ func initShardOptions(c *cfg) {
 		metabaseCfg := sc.Metabase()
 		gcCfg := sc.GC()
 
+		piloramaCfg := sc.Pilorama()
+		piloramaPath := piloramaCfg.Path()
+		if piloramaPath == "" {
+			piloramaPath = filepath.Join(blobStorCfg.Path(), "pilorama.db")
+		}
+
+		piloramaOpts := []pilorama.Option{
+			pilorama.WithPath(piloramaPath),
+			pilorama.WithPerm(piloramaCfg.Perm()),
+			pilorama.WithNoSync(piloramaCfg.NoSync()),
+			pilorama.WithMaxBatchSize(piloramaCfg.MaxBatchSize()),
+			pilorama.WithMaxBatchDelay(piloramaCfg.MaxBatchDelay())}
+
 		metaPath := metabaseCfg.Path()
 		metaPerm := metabaseCfg.BoltDB().Perm()
 		fatalOnErr(util.MkdirAllX(filepath.Dir(metaPath), metaPerm))
@@ -456,6 +470,7 @@ func initShardOptions(c *cfg) {
 					Timeout: 100 * time.Millisecond,
 				}),
 			),
+			shard.WithPiloramaOptions(piloramaOpts...),
 			shard.WithWriteCache(writeCacheCfg.Enabled()),
 			shard.WithWriteCacheOptions(writeCacheOpts...),
 			shard.WithRemoverBatchSize(gcCfg.RemoverBatchSize()),

cmd/neofs-node/config/engine/config_test.go

@@ -8,6 +8,7 @@ import (
 	"github.com/nspcc-dev/neofs-node/cmd/neofs-node/config"
 	engineconfig "github.com/nspcc-dev/neofs-node/cmd/neofs-node/config/engine"
 	shardconfig "github.com/nspcc-dev/neofs-node/cmd/neofs-node/config/engine/shard"
+	piloramaconfig "github.com/nspcc-dev/neofs-node/cmd/neofs-node/config/engine/shard/pilorama"
 	configtest "github.com/nspcc-dev/neofs-node/cmd/neofs-node/config/test"
 	"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/shard"
 	"github.com/stretchr/testify/require"
@@ -53,10 +54,17 @@ func TestEngineSection(t *testing.T) {
 			meta := sc.Metabase()
 			blob := sc.BlobStor()
 			blz := blob.Blobovnicza()
+			pl := sc.Pilorama()
 			gc := sc.GC()
 
 			switch num {
 			case 0:
+				require.Equal(t, "tmp/0/blob/pilorama.db", pl.Path())
+				require.Equal(t, fs.FileMode(piloramaconfig.PermDefault), pl.Perm())
+				require.False(t, pl.NoSync())
+				require.Equal(t, pl.MaxBatchDelay(), 10*time.Millisecond)
+				require.Equal(t, pl.MaxBatchSize(), 200)
+
 				require.Equal(t, false, wc.Enabled())
 
 				require.Equal(t, "tmp/0/cache", wc.Path())
@@ -89,6 +97,12 @@ func TestEngineSection(t *testing.T) {
 				require.Equal(t, false, sc.RefillMetabase())
 				require.Equal(t, shard.ModeReadOnly, sc.Mode())
 			case 1:
+				require.Equal(t, "tmp/1/blob/pilorama.db", pl.Path())
+				require.Equal(t, fs.FileMode(0644), pl.Perm())
+				require.True(t, pl.NoSync())
+				require.Equal(t, 5*time.Millisecond, pl.MaxBatchDelay())
+				require.Equal(t, 100, pl.MaxBatchSize())
+
 				require.Equal(t, true, wc.Enabled())
 
 				require.Equal(t, "tmp/1/cache", wc.Path())

cmd/neofs-node/config/engine/shard/config.go

@@ -7,6 +7,7 @@ import (
 	blobstorconfig "github.com/nspcc-dev/neofs-node/cmd/neofs-node/config/engine/shard/blobstor"
 	gcconfig "github.com/nspcc-dev/neofs-node/cmd/neofs-node/config/engine/shard/gc"
 	metabaseconfig "github.com/nspcc-dev/neofs-node/cmd/neofs-node/config/engine/shard/metabase"
+	piloramaconfig "github.com/nspcc-dev/neofs-node/cmd/neofs-node/config/engine/shard/pilorama"
 	writecacheconfig "github.com/nspcc-dev/neofs-node/cmd/neofs-node/config/engine/shard/writecache"
 	"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/shard"
 )
@@ -44,6 +45,14 @@ func (x *Config) WriteCache() *writecacheconfig.Config {
 	)
 }
 
+// Pilorama returns "pilorama" subsection as a piloramaconfig.Config.
+func (x *Config) Pilorama() *piloramaconfig.Config {
+	return piloramaconfig.From(
+		(*config.Config)(x).
+			Sub("pilorama"),
+	)
+}
+
 // GC returns "gc" subsection as a gcconfig.Config.
 func (x *Config) GC() *gcconfig.Config {
 	return gcconfig.From(

cmd/neofs-node/config/engine/shard/pilorama/config.go

@@ -0,0 +1,70 @@
+package piloramaconfig
+
+import (
+	"io/fs"
+	"time"
+
+	"github.com/nspcc-dev/neofs-node/cmd/neofs-node/config"
+)
+
+// Config is a wrapper over the config section
+// which provides access to Metabase configurations.
+type Config config.Config
+
+const (
+	// PermDefault is a default permission bits for metabase file.
+	PermDefault = 0660
+)
+
+// From wraps config section into Config.
+func From(c *config.Config) *Config {
+	return (*Config)(c)
+}
+
+// Path returns the value of "path" config parameter.
+//
+// Returns empty string if missing, for compatibility with older configurations.
+func (x *Config) Path() string {
+	return config.String((*config.Config)(x), "path")
+}
+
+// Perm returns the value of "perm" config parameter as a fs.FileMode.
+//
+// Returns PermDefault if the value is not a positive number.
+func (x *Config) Perm() fs.FileMode {
+	p := config.UintSafe((*config.Config)(x), "perm")
+	if p == 0 {
+		p = PermDefault
+	}
+
+	return fs.FileMode(p)
+}
+
+// NoSync returns the value of "no_sync" config parameter as a bool value.
+//
+// Returns false if the value is not a boolean.
+func (x *Config) NoSync() bool {
+	return config.BoolSafe((*config.Config)(x), "no_sync")
+}
+
+// MaxBatchDelay returns the value of "max_batch_delay" config parameter.
+//
+// Returns 0 if the value is not a positive number.
+func (x *Config) MaxBatchDelay() time.Duration {
+	d := config.DurationSafe((*config.Config)(x), "max_batch_delay")
+	if d <= 0 {
+		d = 0
+	}
+	return d
+}
+
+// MaxBatchSize returns the value of "max_batch_size" config parameter.
+//
+// Returns 0 if the value is not a positive number.
+func (x *Config) MaxBatchSize() int {
+	s := int(config.IntSafe((*config.Config)(x), "max_batch_size"))
+	if s <= 0 {
+		s = 0
+	}
+	return s
+}

config/example/node.env

@@ -107,6 +107,10 @@ NEOFS_STORAGE_SHARD_0_BLOBSTOR_BLOBOVNICZA_SIZE=4194304
 NEOFS_STORAGE_SHARD_0_BLOBSTOR_BLOBOVNICZA_DEPTH=1
 NEOFS_STORAGE_SHARD_0_BLOBSTOR_BLOBOVNICZA_WIDTH=4
 NEOFS_STORAGE_SHARD_0_BLOBSTOR_BLOBOVNICZA_OPENED_CACHE_CAPACITY=50
+### Pilorama config
+NEOFS_STORAGE_SHARD_0_PILORAMA_PATH="tmp/0/blob/pilorama.db"
+NEOFS_STORAGE_SHARD_0_PILORAMA_MAX_BATCH_DELAY=10ms
+NEOFS_STORAGE_SHARD_0_PILORAMA_MAX_BATCH_SIZE=200
 ### GC config
 #### Limit of the single data remover's batching operation in number of objects
 NEOFS_STORAGE_SHARD_0_GC_REMOVER_BATCH_SIZE=150
@@ -142,6 +146,12 @@ NEOFS_STORAGE_SHARD_1_BLOBSTOR_BLOBOVNICZA_SIZE=4194304
 NEOFS_STORAGE_SHARD_1_BLOBSTOR_BLOBOVNICZA_DEPTH=1
 NEOFS_STORAGE_SHARD_1_BLOBSTOR_BLOBOVNICZA_WIDTH=4
 NEOFS_STORAGE_SHARD_1_BLOBSTOR_BLOBOVNICZA_OPENED_CACHE_CAPACITY=50
+### Pilorama config
+NEOFS_STORAGE_SHARD_1_PILORAMA_PATH="tmp/1/blob/pilorama.db"
+NEOFS_STORAGE_SHARD_1_PILORAMA_PERM=0644
+NEOFS_STORAGE_SHARD_1_PILORAMA_NO_SYNC=true
+NEOFS_STORAGE_SHARD_1_PILORAMA_MAX_BATCH_DELAY=5ms
+NEOFS_STORAGE_SHARD_1_PILORAMA_MAX_BATCH_SIZE=100
 ### GC config
 #### Limit of the single data remover's batching operation in number of objects
 NEOFS_STORAGE_SHARD_1_GC_REMOVER_BATCH_SIZE=200

config/example/node.json

@@ -158,6 +158,11 @@
             "opened_cache_capacity": 50
           }
         },
+        "pilorama": {
+          "path": "tmp/0/blob/pilorama.db",
+          "max_batch_delay": "10ms",
+          "max_batch_size": 200
+        },
         "gc": {
           "remover_batch_size": 150,
           "remover_sleep_interval": "2m"
@@ -194,6 +199,13 @@
             "opened_cache_capacity": 50
           }
         },
+        "pilorama": {
+          "path": "tmp/1/blob/pilorama.db",
+          "perm": "0644",
+          "no_sync": true,
+          "max_batch_delay": "5ms",
+          "max_batch_size": 100
+        },
         "gc": {
           "remover_batch_size": 200,
           "remover_sleep_interval": "5m"

config/example/node.yaml

@@ -120,6 +120,10 @@ storage:
         max_batch_size: 200
         max_batch_delay: 20ms
 
+      pilorama:
+        max_batch_delay: 5ms # maximum delay for a batch of operations to be executed
+        max_batch_size: 100 # maximum amount of operations in a single batch
+
       blobstor:
         compress: false  # turn on/off zstd(level 3) compression of stored objects
         perm: 0644  # permissions for blobstor files(directories: +x for current user and group)
@@ -157,6 +161,11 @@ storage:
           - audio/*
           - video/*
 
+      pilorama:
+        path: tmp/0/blob/pilorama.db # path to the pilorama database. If omitted, `pilorama.db` file is created blobstor.path
+        max_batch_delay: 10ms
+        max_batch_size: 200
+
       gc:
         remover_batch_size: 150  # number of objects to be removed by the garbage collector
         remover_sleep_interval: 2m  # frequency of the garbage collector invocation
@@ -171,3 +180,9 @@ storage:
 
       blobstor:
         path: tmp/1/blob  # blobstor path
+
+
+      pilorama:
+        path: tmp/1/blob/pilorama.db
+        no_sync: true # USE WITH CAUTION. Return to user before pages have been persisted.
+        perm: 0644 # permission to use for the database file and intermediate directories

pkg/local_object_storage/engine/engine_test.go

@@ -8,6 +8,7 @@ import (
 
 	"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor"
 	meta "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/metabase"
+	"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/pilorama"
 	"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/shard"
 	"github.com/nspcc-dev/neofs-sdk-go/checksum"
 	checksumtest "github.com/nspcc-dev/neofs-sdk-go/checksum/test"
@@ -99,6 +100,7 @@ func testNewShard(t testing.TB, id int) *shard.Shard {
 			blobstor.WithBlobovniczaShallowDepth(2),
 			blobstor.WithRootPerm(0700),
 		),
+		shard.WithPiloramaOptions(pilorama.WithPath(filepath.Join(t.Name(), fmt.Sprintf("%d.pilorama", id)))),
 		shard.WithMetaBaseOptions(
 			meta.WithPath(filepath.Join(t.Name(), fmt.Sprintf("%d.metabase", id))),
 			meta.WithPermissions(0700),
@@ -123,7 +125,10 @@ func testEngineFromShardOpts(t *testing.T, num int, extraOpts func(int) []shard.
 			shard.WithMetaBaseOptions(
 				meta.WithPath(filepath.Join(t.Name(), fmt.Sprintf("metabase%d", i))),
 				meta.WithPermissions(0700),
-			)}, extraOpts(i)...)...)
+			),
+			shard.WithPiloramaOptions(
+				pilorama.WithPath(filepath.Join(t.Name(), fmt.Sprintf("pilorama%d", i)))),
+		}, extraOpts(i)...)...)
 		require.NoError(t, err)
 	}
 

pkg/local_object_storage/engine/error_test.go

@@ -10,6 +10,7 @@ import (
 	"github.com/nspcc-dev/neofs-node/pkg/core/object"
 	"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor"
 	meta "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/metabase"
+	"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/pilorama"
 	"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/shard"
 	apistatus "github.com/nspcc-dev/neofs-sdk-go/client/status"
 	cidtest "github.com/nspcc-dev/neofs-sdk-go/container/id/test"
@@ -49,7 +50,10 @@ func newEngineWithErrorThreshold(t testing.TB, dir string, errThreshold uint32)
 				blobstor.WithRootPerm(0700)),
 			shard.WithMetaBaseOptions(
 				meta.WithPath(filepath.Join(dir, fmt.Sprintf("%d.metabase", i))),
-				meta.WithPermissions(0700)))
+				meta.WithPermissions(0700)),
+			shard.WithPiloramaOptions(
+				pilorama.WithPath(filepath.Join(dir, fmt.Sprintf("%d.pilorama", i))),
+				pilorama.WithPerm(0700)))
 		require.NoError(t, err)
 	}
 	require.NoError(t, e.Open())

pkg/local_object_storage/pilorama/boltdb.go

@@ -3,22 +3,22 @@ package pilorama
 import (
 	"bytes"
 	"encoding/binary"
+	"fmt"
 	"math/rand"
 	"os"
 	"path/filepath"
 
 	"github.com/nspcc-dev/neo-go/pkg/io"
+	"github.com/nspcc-dev/neofs-node/pkg/util"
 	cidSDK "github.com/nspcc-dev/neofs-sdk-go/container/id"
 	"go.etcd.io/bbolt"
 )
 
 type boltForest struct {
-	path string
 	db *bbolt.DB
+	cfg
 }
 
-const defaultMaxBatchSize = 10
-
 var (
 	dataBucket = []byte{0}
 	logBucket  = []byte{1}
@@ -41,25 +41,41 @@ var (
 // 'm' + node (id) -> serialized meta
 // 'c' + parent (id) + child (id) -> 0/1
 // 'i' + 0 + attrKey + 0 + attrValue + 0 + parent (id) + node (id) -> 0/1 (1 for automatically created nodes)
-func NewBoltForest(path string) ForestStorage {
+func NewBoltForest(opts ...Option) ForestStorage {
-	return &boltForest{path: path}
+	b := boltForest{
+		cfg: cfg{
+			perm:          os.ModePerm,
+			maxBatchDelay: bbolt.DefaultMaxBatchDelay,
+			maxBatchSize:  bbolt.DefaultMaxBatchSize,
+		},
+	}
+
+	for i := range opts {
+		opts[i](&b.cfg)
+	}
+
+	return &b
 }
 
 func (t *boltForest) Init() error { return nil }
 func (t *boltForest) Open() error {
-	if err := os.MkdirAll(filepath.Dir(t.path), os.ModePerm); err != nil {
+	err := util.MkdirAllX(filepath.Dir(t.path), t.perm)
-		return err
-	}
-
-	db, err := bbolt.Open(t.path, os.ModePerm, bbolt.DefaultOptions)
 	if err != nil {
-		return err
+		return fmt.Errorf("can't create dir %s for the pilorama: %w", t.path, err)
 	}
 
-	db.MaxBatchSize = defaultMaxBatchSize
+	opts := *bbolt.DefaultOptions
-	t.db = db
+	opts.NoSync = t.noSync
 
-	return db.Update(func(tx *bbolt.Tx) error {
+	t.db, err = bbolt.Open(t.path, t.perm, &opts)
+	if err != nil {
+		return fmt.Errorf("can't open the pilorama DB: %w", err)
+	}
+
+	t.db.MaxBatchSize = t.maxBatchSize
+	t.db.MaxBatchDelay = t.maxBatchDelay
+
+	return t.db.Update(func(tx *bbolt.Tx) error {
 		_, err := tx.CreateBucketIfNotExists(dataBucket)
 		if err != nil {
 			return err

pkg/local_object_storage/pilorama/forest_test.go

@@ -31,7 +31,7 @@ var providers = []struct {
 		tmpDir, err := os.MkdirTemp(os.TempDir(), "*")
 		require.NoError(t, err)
 
-		f := NewBoltForest(filepath.Join(tmpDir, "test.db"))
+		f := NewBoltForest(WithPath(filepath.Join(tmpDir, "test.db")))
 		require.NoError(t, f.Init())
 		require.NoError(t, f.Open())
 		t.Cleanup(func() {

pkg/local_object_storage/pilorama/option.go

@@ -0,0 +1,46 @@
+package pilorama
+
+import (
+	"io/fs"
+	"time"
+)
+
+type Option func(*cfg)
+
+type cfg struct {
+	path          string
+	perm          fs.FileMode
+	noSync        bool
+	maxBatchDelay time.Duration
+	maxBatchSize  int
+}
+
+func WithPath(path string) Option {
+	return func(c *cfg) {
+		c.path = path
+	}
+}
+
+func WithPerm(perm fs.FileMode) Option {
+	return func(c *cfg) {
+		c.perm = perm
+	}
+}
+
+func WithNoSync(noSync bool) Option {
+	return func(c *cfg) {
+		c.noSync = noSync
+	}
+}
+
+func WithMaxBatchDelay(d time.Duration) Option {
+	return func(c *cfg) {
+		c.maxBatchDelay = d
+	}
+}
+
+func WithMaxBatchSize(size int) Option {
+	return func(c *cfg) {
+		c.maxBatchSize = size
+	}
+}

pkg/local_object_storage/shard/control_test.go

@@ -9,6 +9,7 @@ import (
 	"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor"
 	"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor/fstree"
 	meta "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/metabase"
+	"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/pilorama"
 	apistatus "github.com/nspcc-dev/neofs-sdk-go/client/status"
 	cid "github.com/nspcc-dev/neofs-sdk-go/container/id"
 	cidtest "github.com/nspcc-dev/neofs-sdk-go/container/id/test"
@@ -31,6 +32,7 @@ func TestRefillMetabaseCorrupted(t *testing.T) {
 
 	sh := New(
 		WithBlobStorOptions(blobOpts...),
+		WithPiloramaOptions(pilorama.WithPath(filepath.Join(dir, "pilorama"))),
 		WithMetaBaseOptions(meta.WithPath(filepath.Join(dir, "meta"))))
 	require.NoError(t, sh.Open())
 	require.NoError(t, sh.Init())
@@ -55,6 +57,7 @@ func TestRefillMetabaseCorrupted(t *testing.T) {
 
 	sh = New(
 		WithBlobStorOptions(blobOpts...),
+		WithPiloramaOptions(pilorama.WithPath(filepath.Join(dir, "pilorama"))),
 		WithMetaBaseOptions(meta.WithPath(filepath.Join(dir, "meta_new"))),
 		WithRefillMetabase(true))
 	require.NoError(t, sh.Open())
@@ -83,6 +86,8 @@ func TestRefillMetabase(t *testing.T) {
 		WithMetaBaseOptions(
 			meta.WithPath(filepath.Join(p, "meta")),
 		),
+		WithPiloramaOptions(
+			pilorama.WithPath(filepath.Join(p, "pilorama"))),
 	)
 
 	// open Blobstor
@@ -246,6 +251,8 @@ func TestRefillMetabase(t *testing.T) {
 		WithMetaBaseOptions(
 			meta.WithPath(filepath.Join(p, "meta_restored")),
 		),
+		WithPiloramaOptions(
+			pilorama.WithPath(filepath.Join(p, "pilorama_another"))),
 	)
 
 	// open Blobstor

pkg/local_object_storage/shard/shard.go

@@ -2,7 +2,6 @@ package shard
 
 import (
 	"context"
-	"path/filepath"
 	"sync"
 	"time"
 
@@ -62,6 +61,8 @@ type cfg struct {
 
 	writeCacheOpts []writecache.Option
 
+	piloramaOpts []pilorama.Option
+
 	log *logger.Logger
 
 	gcCfg *gcCfg
@@ -108,7 +109,7 @@ func New(opts ...Option) *Shard {
 		metaBase:   mb,
 		writeCache: writeCache,
 		tsSource:   c.tsSource,
-		pilorama:   pilorama.NewBoltForest(filepath.Join(bs.DumpInfo().RootPath, "pilorama.db")),
+		pilorama:   pilorama.NewBoltForest(c.piloramaOpts...),
 	}
 
 	s.fillInfo()
@@ -144,6 +145,13 @@ func WithWriteCacheOptions(opts ...writecache.Option) Option {
 	}
 }
 
+// WithPiloramaOptions returns option to set internal write cache options.
+func WithPiloramaOptions(opts ...pilorama.Option) Option {
+	return func(c *cfg) {
+		c.piloramaOpts = opts
+	}
+}
+
 // WithLogger returns option to set Shard's logger.
 func WithLogger(l *logger.Logger) Option {
 	return func(c *cfg) {

pkg/local_object_storage/shard/shard_test.go

@@ -10,6 +10,7 @@ import (
 
 	"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor"
 	meta "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/metabase"
+	"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/pilorama"
 	"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/shard"
 	"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/writecache"
 	"github.com/nspcc-dev/neofs-sdk-go/checksum"
@@ -49,6 +50,7 @@ func newCustomShard(t testing.TB, rootPath string, enableWriteCache bool, wcOpts
 		shard.WithMetaBaseOptions(
 			meta.WithPath(filepath.Join(rootPath, "meta")),
 		),
+		shard.WithPiloramaOptions(pilorama.WithPath(filepath.Join(rootPath, "pilorama"))),
 		shard.WithWriteCache(enableWriteCache),
 		shard.WithWriteCacheOptions(
 			append(