[#421] Create common interface for kv repositories

Signed-off-by: Alejandro Lopez <a.lopez@yadro.com>

cmd/frostfs-lens/internal/writecache/list.go

@@ -5,7 +5,7 @@ import (
 	"io"
 
 	common "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-lens/internal"
-	"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/writecache"
+	"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/kvio"
 	oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
 	"github.com/spf13/cobra"
 )
@@ -25,14 +25,17 @@ func listFunc(cmd *cobra.Command, _ []string) {
 	// other targets can be supported
 	w := cmd.OutOrStderr()
 
-	wAddr := func(addr oid.Address) error {
-		_, err := io.WriteString(w, fmt.Sprintf("%s\n", addr))
-		return err
-	}
-
 	db := openWC(cmd)
 	defer db.Close()
 
-	err := writecache.IterateDB(db, wAddr)
+	err := kvio.ForEachKey(db, func(k kvio.Key) error {
+		var addr oid.Address
+		if err := addr.DecodeString(string(k)); err != nil {
+			return fmt.Errorf("could not parse object address: %w", err)
+		}
+		_, err := io.WriteString(w, fmt.Sprintf("%s\n", addr))
+		return err
+	})
+
 	common.ExitOnErr(cmd, common.Errf("write-cache iterator failure: %w", err))
 }

cmd/frostfs-lens/internal/writecache/root.go

@@ -2,9 +2,11 @@ package writecache
 
 import (
 	"os"
+	"time"
 
 	common "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-lens/internal"
-	"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/writecache"
+	"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/kvio"
+	bboltrepo "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/kvio/bbolt"
 	"github.com/spf13/cobra"
 	"go.etcd.io/bbolt"
 )
@@ -25,9 +27,15 @@ func init() {
 	Root.AddCommand(listCMD, inspectCMD)
 }
 
-func openWC(cmd *cobra.Command) *bbolt.DB {
+func openWC(cmd *cobra.Command) kvio.Repository {
-	db, err := writecache.OpenDB(vPath, true, os.OpenFile)
+	db, err := bboltrepo.Open(vPath, &bboltrepo.Options{
+		Options: bbolt.Options{
+			NoFreelistSync: true,
+			ReadOnly:       true,
+			Timeout:        100 * time.Millisecond,
+			OpenFile:       os.OpenFile,
+		},
+	})
 	common.ExitOnErr(cmd, common.Errf("could not open write-cache db: %w", err))
-
 	return db
 }

go.mod

@@ -41,6 +41,17 @@ require (
 	gopkg.in/yaml.v3 v3.0.1
 )
 
+require (
+	github.com/dgraph-io/ristretto v0.1.1 // indirect
+	github.com/dustin/go-humanize v1.0.0 // indirect
+	github.com/gogo/protobuf v1.3.2 // indirect
+	github.com/golang/glog v1.1.0 // indirect
+	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
+	github.com/google/flatbuffers v1.12.1 // indirect
+	github.com/pkg/errors v0.9.1 // indirect
+	go.opencensus.io v0.24.0 // indirect
+)
+
 require (
 	git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0 // indirect
 	git.frostfs.info/TrueCloudLab/rfc6979 v0.4.0 // indirect
@@ -54,6 +65,7 @@ require (
 	github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0 // indirect
+	github.com/dgraph-io/badger/v4 v4.1.0
 	github.com/fsnotify/fsnotify v1.6.0 // indirect
 	github.com/go-logr/logr v1.2.4 // indirect
 	github.com/go-logr/stdr v1.2.2 // indirect

pkg/local_object_storage/kvio/badger/badger.go

@@ -0,0 +1,58 @@
+// Package badger provides a kvio.Repository backed by badger.
+package badger
+
+import (
+	"fmt"
+
+	"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/kvio"
+	badger "github.com/dgraph-io/badger/v4"
+)
+
+type impl struct {
+	db *badger.DB
+}
+
+func Open(opts badger.Options) (kvio.Repository, error) {
+	db, err := badger.Open(opts)
+	if err != nil {
+		return nil, fmt.Errorf("could not open database: %w", err)
+	}
+
+	return impl{db}, nil
+}
+
+func (r impl) Read(f func(kvio.ReadOnlyTx) error) error {
+	return r.db.View(func(tx *badger.Txn) error {
+		return f(txImpl{tx})
+	})
+}
+
+func (r impl) Write(f func(kvio.WriteOnlyTx) error) error {
+	wb := r.db.NewWriteBatch()
+	defer wb.Cancel()
+	if err := f(writeBatchTx{wb}); err != nil {
+		return err
+	}
+	return wb.Flush()
+}
+
+func (r impl) ReadWrite(f func(kvio.ReadWriteTx) error) error {
+	return r.db.Update(func(tx *badger.Txn) error {
+		return f(txImpl{tx})
+	})
+}
+
+func (r impl) Stats() (kvio.Stats, error) {
+	var stats kvio.Stats
+	err := r.Read(func(tx kvio.ReadOnlyTx) error {
+		cur := tx.IterateKeys()
+		defer cur.Close()
+		for ; cur.Key() != nil; cur.Next() {
+			stats.KeyCount++
+		}
+		return nil
+	})
+	return stats, err
+}
+
+func (r impl) Close() error { return r.db.Close() }

pkg/local_object_storage/kvio/badger/cursor.go

@@ -0,0 +1,31 @@
+package badger
+
+import (
+	"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/kvio"
+	badger "github.com/dgraph-io/badger/v4"
+)
+
+type cursor struct {
+	it *badger.Iterator
+}
+
+func (c cursor) Key() kvio.Key {
+	if c.it.Valid() {
+		return c.it.Item().Key()
+	}
+	return nil
+}
+
+func (c cursor) Rewind()         { c.it.Rewind() }
+func (c cursor) Seek(k kvio.Key) { c.it.Seek(k) }
+func (c cursor) Next()           { c.it.Next() }
+func (c cursor) Close()          { c.it.Close() }
+
+func (c cursor) Value(f func(kvio.Value) error) error {
+	if c.it.Valid() {
+		return c.it.Item().Value(func(val []byte) error {
+			return f(val)
+		})
+	}
+	return f(nil)
+}

pkg/local_object_storage/kvio/badger/tx.go

@@ -0,0 +1,45 @@
+package badger
+
+import (
+	"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/kvio"
+	badger "github.com/dgraph-io/badger/v4"
+)
+
+type txImpl struct {
+	tx *badger.Txn
+}
+
+func (tx txImpl) Read(k kvio.Key, f func(kvio.Value) error) error {
+	it, err := tx.tx.Get(k)
+	if err != nil {
+		if err == badger.ErrKeyNotFound {
+			return f(nil)
+		}
+		return err
+	}
+	return it.Value(func(val []byte) error { return f(val) })
+}
+
+func (tx txImpl) Write(k kvio.Key, v kvio.Value) error { return tx.tx.Set(k, v) }
+func (tx txImpl) Delete(k kvio.Key) error              { return tx.tx.Delete(k) }
+
+func (tx txImpl) IterateKeys() kvio.KeyCursor {
+	opts := badger.DefaultIteratorOptions
+	opts.PrefetchValues = false
+	cur := tx.tx.NewIterator(opts)
+	cur.Rewind()
+	return cursor{cur}
+}
+
+func (tx txImpl) Iterate() kvio.Cursor {
+	cur := tx.tx.NewIterator(badger.DefaultIteratorOptions)
+	cur.Rewind()
+	return cursor{cur}
+}
+
+type writeBatchTx struct {
+	wb *badger.WriteBatch
+}
+
+func (tx writeBatchTx) Write(k kvio.Key, v kvio.Value) error { return tx.wb.Set(k, v) }
+func (tx writeBatchTx) Delete(k kvio.Key) error              { return tx.wb.Delete(k) }

pkg/local_object_storage/kvio/bbolt/bbolt.go

@@ -0,0 +1,105 @@
+// Package bbolt provides a kvio.Repository backed by bbolt.
+package bbolt
+
+import (
+	"errors"
+	"fmt"
+	"os"
+	"time"
+
+	"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/kvio"
+	"go.etcd.io/bbolt"
+)
+
+type impl struct {
+	db *bbolt.DB
+}
+
+type Options struct {
+	bbolt.Options
+	MaxBatchSize  int
+	MaxBatchDelay time.Duration
+}
+
+var (
+	defaultBucket = []byte{0}
+
+	// ErrNoDefaultBucket is returned when the default bucket for objects is missing.
+	ErrNoDefaultBucket = errors.New("no default bucket")
+)
+
+func Open(path string, opts *Options) (kvio.Repository, error) {
+	db, err := bbolt.Open(path, os.ModePerm, &opts.Options)
+	if err != nil {
+		return nil, fmt.Errorf("could not open database: %w", err)
+	}
+
+	if opts.MaxBatchSize > 0 {
+		db.MaxBatchSize = opts.MaxBatchSize
+	}
+	if opts.MaxBatchDelay > 0 {
+		db.MaxBatchDelay = opts.MaxBatchDelay
+	}
+
+	if !opts.Options.ReadOnly {
+		err = db.Update(func(tx *bbolt.Tx) error {
+			_, err := tx.CreateBucketIfNotExists(defaultBucket)
+			return err
+		})
+		if err != nil {
+			return nil, fmt.Errorf("could not create default bucket: %w", err)
+		}
+	}
+
+	return impl{db}, nil
+}
+
+func (r impl) Read(f func(kvio.ReadOnlyTx) error) error {
+	return r.db.View(func(tx *bbolt.Tx) error {
+		b := tx.Bucket(defaultBucket)
+		if b == nil {
+			return ErrNoDefaultBucket
+		}
+		return f(txImpl{b, tx})
+	})
+}
+
+func (r impl) Write(f func(kvio.WriteOnlyTx) error) error {
+	return r.db.Batch(func(tx *bbolt.Tx) error {
+		b := tx.Bucket(defaultBucket)
+		if b == nil {
+			return ErrNoDefaultBucket
+		}
+		return f(txImpl{b, tx})
+	})
+}
+
+func (r impl) ReadWrite(f func(kvio.ReadWriteTx) error) error {
+	return r.db.Update(func(tx *bbolt.Tx) error {
+		b := tx.Bucket(defaultBucket)
+		if b == nil {
+			return ErrNoDefaultBucket
+		}
+		return f(txImpl{b, tx})
+	})
+}
+
+func (r impl) Stats() (kvio.Stats, error) {
+	var keyCount uint64
+	err := r.db.View(func(tx *bbolt.Tx) error {
+		b := tx.Bucket(defaultBucket)
+		if b != nil {
+			keyCount = uint64(b.Stats().KeyN)
+		}
+		return nil
+	})
+	if err != nil {
+		return kvio.Stats{}, fmt.Errorf("could not read write-cache DB counter: %w", err)
+	}
+
+	return kvio.Stats{
+		KeyCount: keyCount,
+	}, nil
+}
+
+func (r impl) Close() error { return r.db.Close() }

pkg/local_object_storage/kvio/bbolt/cursor.go

@@ -0,0 +1,19 @@
+package bbolt
+
+import (
+	"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/kvio"
+	"go.etcd.io/bbolt"
+)
+
+type cursor struct {
+	cur *bbolt.Cursor
+	key []byte
+	val []byte
+}
+
+func (c *cursor) Key() kvio.Key                        { return c.key }
+func (c *cursor) Rewind()                              { c.key, c.val = c.cur.First() }
+func (c *cursor) Seek(k kvio.Key)                      { c.key, c.val = c.cur.Seek(k) }
+func (c *cursor) Next()                                { c.key, c.val = c.cur.Next() }
+func (c *cursor) Close()                               {}
+func (c *cursor) Value(f func(kvio.Value) error) error { return f(c.val) }

pkg/local_object_storage/kvio/bbolt/tx.go

@@ -0,0 +1,21 @@
+package bbolt
+
+import (
+	"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/kvio"
+	"go.etcd.io/bbolt"
+)
+
+type txImpl struct {
+	b  *bbolt.Bucket
+	tx *bbolt.Tx
+}
+
+func (tx txImpl) Read(k kvio.Key, f func(kvio.Value) error) error { return f(tx.b.Get(k)) }
+func (tx txImpl) Write(k kvio.Key, v kvio.Value) error            { return tx.b.Put(k, v) }
+func (tx txImpl) Delete(k kvio.Key) error                         { return tx.b.Delete(k) }
+func (tx txImpl) IterateKeys() kvio.KeyCursor                     { return tx.Iterate() }
+func (tx txImpl) Iterate() kvio.Cursor {
+	cur := tx.b.Cursor()
+	k0, v0 := cur.First()
+	return &cursor{cur, k0, v0}
+}

pkg/local_object_storage/kvio/kvio.go

@@ -0,0 +1,88 @@
+// Package kvio provides basic interfaces to key-value I/O primitives.
+// It serves as a wrapper for exisiting implementations such as those
+// provided by popular key-value stores (bbolt, badger, leveldb, etc.)
+// and can serve to compose them as well.
+package kvio
+
+type (
+	Key   []byte
+	Value []byte
+)
+
+// Writer is the interface that wraps the Write method for a key-value entry.
+type Writer interface {
+	Write(Key, Value) error
+}
+
+// Reader is the interface that wraps the Read method for a key-value entry.
+type Reader interface {
+	Read(Key, func(Value) error) error
+}
+
+// Deleter is the interface that wraps the Delete method for a key-value entry.
+type Deleter interface {
+	Delete(Key) error
+}
+
+// KeyCursor is the interface of cursors that can iterate over a set of keys.
+type KeyCursor interface {
+	// Key returns the key of the current entry, or nil if at the end of the cursor.
+	Key() Key
+	// Rewind rewinds the cursor to the first key in lexicographical order.
+	Rewind()
+	// Seek moves the cursor to the smallest key that is equal or greater than the
+	// provided key in lexicographical order.
+	Seek(Key)
+	// Next moves the cursor to the next key in lexicographical order.
+	Next()
+	// Close frees the resources allocated by the cursor and invalidates it.
+	Close()
+}
+
+// Cursor is the interface of cursors that can iterate over a set of key-value entries.
+type Cursor interface {
+	KeyCursor
+	// Value calls the given function with the value of the current entry, or nil if
+	// at the end of the cursor.
+	Value(func(Value) error) error
+}
+
+// Iterator is the interface which provides cursors over sets of key-value entries.
+type Iterator interface {
+	// IterateKeys provides a cursor to iterate the key set in lexicographical order.
+	IterateKeys() KeyCursor
+	// IterateKeys provides a cursor to iterate the key-value set in lexicographical order of keys.
+	Iterate() Cursor
+}
+
+// ReadOnlyTx is the interface of read-only transactions for key-value repositories.
+type ReadOnlyTx interface {
+	Reader
+	Iterator
+}
+
+// WriteOnlyTx is the interface of write-only transactions for key-value repositories.
+type WriteOnlyTx interface {
+	Writer
+	Deleter
+}
+
+// ReadWriteTx is the interface of read-write transactions for key-value repositories.
+type ReadWriteTx interface {
+	ReadOnlyTx
+	WriteOnlyTx
+}
+
+// Stats are the common statistics for key-value repositories.
+type Stats struct {
+	KeyCount uint64
+}
+
+// Repository is the interface of key-value repositories.
+type Repository interface {
+	Read(func(ReadOnlyTx) error) error
+	Write(func(WriteOnlyTx) error) error
+	ReadWrite(func(ReadWriteTx) error) error
+	Stats() (Stats, error)
+	Close() error
+}

pkg/local_object_storage/kvio/util.go

@@ -0,0 +1,70 @@
+package kvio
+
+import "github.com/nspcc-dev/neo-go/pkg/util/slice"
+
+// ForEachKey calls the given function for each key in the given repository in lexicographical order.
+func ForEachKey(repo Repository, f func(Key) error) error {
+	return repo.Read(func(tx ReadOnlyTx) error {
+		cur := tx.IterateKeys()
+		defer cur.Close()
+		for ; cur.Key() != nil; cur.Next() {
+			if err := f(cur.Key()); err != nil {
+				return err
+			}
+		}
+		return nil
+	})
+}
+
+// ForEach calls the given function for each key-value entry in the given repository
+// in lexicographical order of the keys.
+func ForEach(repo Repository, f func(Key, Value) error) error {
+	return repo.Read(func(tx ReadOnlyTx) error {
+		cur := tx.Iterate()
+		defer cur.Close()
+		for ; cur.Key() != nil; cur.Next() {
+			var val Value
+			if err := cur.Value(func(v Value) error {
+				val = v
+				return nil
+			}); err != nil {
+				return err
+			}
+			if err := f(cur.Key(), val); err != nil {
+				return err
+			}
+		}
+		return nil
+	})
+}
+
+// Write writes a single key-value entry to the given repository.
+// This is a convenience method that takes care of creating the appropriate transaction.
+func Write(repo Repository, k Key, v Value) error {
+	return repo.Write(func(tx WriteOnlyTx) error {
+		return tx.Write(k, v)
+	})
+}
+
+// Read reads a single key-value entry from the given repository.
+// This is a convenience method that takes care of creating the appropriate transaction.
+func Read(repo Repository, k Key) (Value, error) {
+	var ret Value
+	err := repo.Read(func(tx ReadOnlyTx) error {
+		return tx.Read(k, func(v Value) error {
+			if v != nil {
+				ret = slice.Copy(v)
+			}
+			return nil
+		})
+	})
+	return ret, err
+}
+
+// Deletes deletes a single key-value entry from the given repository.
+// This is a convenience method that takes care of creating the appropriate transaction.
+func Delete(repo Repository, k Key) error {
+	return repo.Write(func(tx WriteOnlyTx) error {
+		return tx.Delete(k)
+	})
+}

pkg/local_object_storage/writecache/delete.go

@@ -7,9 +7,9 @@ import (
 	"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common"
 	storagelog "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/log"
 	"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/metaerr"
+	"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/kvio"
 	"git.frostfs.info/TrueCloudLab/frostfs-observability/tracing"
 	oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
-	"go.etcd.io/bbolt"
 	"go.opentelemetry.io/otel/attribute"
 	"go.opentelemetry.io/otel/trace"
 )
@@ -40,20 +40,16 @@ func (c *cache) Delete(ctx context.Context, addr oid.Address) error {
 	saddr := addr.EncodeToString()
 
 	var dataSize int
-	_ = c.db.View(func(tx *bbolt.Tx) error {
+	_ = c.repo.Read(func(tx kvio.ReadOnlyTx) error {
-		b := tx.Bucket(defaultBucket)
+		return tx.Read([]byte(saddr), func(v kvio.Value) error {
-		dataSize = len(b.Get([]byte(saddr)))
+			dataSize = len(v)
 			return nil
 		})
+	})
 
 	if dataSize > 0 {
 		storageType = StorageTypeDB
-		err := c.db.Update(func(tx *bbolt.Tx) error {
+		if err := kvio.Delete(c.repo, []byte(saddr)); err != nil {
-			b := tx.Bucket(defaultBucket)
-			err := b.Delete([]byte(saddr))
-			return err
-		})
-		if err != nil {
 			return err
 		}
 		storagelog.Write(c.log,

pkg/local_object_storage/writecache/flush.go

@@ -11,13 +11,13 @@ import (
 	"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor"
 	"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common"
 	"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/metaerr"
+	"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/kvio"
 	meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase"
 	"git.frostfs.info/TrueCloudLab/frostfs-observability/tracing"
 	"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
 	oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
 	"github.com/mr-tron/base58"
 	"github.com/nspcc-dev/neo-go/pkg/util/slice"
-	"go.etcd.io/bbolt"
 	"go.opentelemetry.io/otel/attribute"
 	"go.opentelemetry.io/otel/trace"
 	"go.uber.org/zap"
@@ -97,32 +97,33 @@ func (c *cache) flushSmallObjects() {
 		}
 
 		// We put objects in batches of fixed size to not interfere with main put cycle a lot.
-		_ = c.db.View(func(tx *bbolt.Tx) error {
+		_ = c.repo.Read(func(tx kvio.ReadOnlyTx) error {
-			b := tx.Bucket(defaultBucket)
+			cur := tx.Iterate()
-			cs := b.Cursor()
+			defer cur.Close()
 
-			var k, v []byte
+			if len(lastKey) != 0 {
+				cur.Seek(lastKey)
+				if bytes.Equal(cur.Key(), lastKey) {
+					cur.Next()
+				}
+			}
 
-			if len(lastKey) == 0 {
+			for ; cur.Key() != nil && len(m) < flushBatchSize; cur.Next() {
-				k, v = cs.First()
+				if len(lastKey) == len(cur.Key()) {
+					copy(lastKey, cur.Key())
 				} else {
-				k, v = cs.Seek(lastKey)
+					lastKey = slice.Copy(cur.Key())
-				if bytes.Equal(k, lastKey) {
-					k, v = cs.Next()
-				}
-			}
-
-			for ; k != nil && len(m) < flushBatchSize; k, v = cs.Next() {
-				if len(lastKey) == len(k) {
-					copy(lastKey, k)
-				} else {
-					lastKey = slice.Copy(k)
 				}
 
+				if err := cur.Value(func(v kvio.Value) error {
 					m = append(m, objectInfo{
-					addr: string(k),
+						addr: string(cur.Key()),
 						data: slice.Copy(v),
 					})
+					return nil
+				}); err != nil {
+					return err
+				}
 			}
 			return nil
 		})
@@ -307,34 +308,26 @@ func (c *cache) flush(ctx context.Context, ignoreErrors bool) error {
 		return err
 	}
 
-	return c.db.View(func(tx *bbolt.Tx) error {
+	return kvio.ForEach(c.repo, func(k kvio.Key, v kvio.Value) error {
 		var addr oid.Address
-
-		b := tx.Bucket(defaultBucket)
-		cs := b.Cursor()
-		for k, data := cs.Seek(nil); k != nil; k, data = cs.Next() {
 		sa := string(k)
 		if err := addr.DecodeString(sa); err != nil {
 			c.reportFlushError("can't decode object address from the DB", sa, metaerr.Wrap(err))
 			if ignoreErrors {
-					continue
+				return nil
 			}
 			return err
 		}
 
 		var obj object.Object
-			if err := obj.Unmarshal(data); err != nil {
+		if err := obj.Unmarshal(v); err != nil {
 			c.reportFlushError("can't unmarshal an object from the DB", sa, metaerr.Wrap(err))
 			if ignoreErrors {
-					continue
+				return nil
 			}
 			return err
 		}
 
-			if err := c.flushObject(ctx, &obj, data, StorageTypeDB); err != nil {
+		return c.flushObject(ctx, &obj, v, StorageTypeDB)
-				return err
-			}
-		}
-		return nil
 	})
 }

pkg/local_object_storage/writecache/flush_test.go

@@ -11,6 +11,7 @@ import (
 	"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor"
 	"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common"
 	"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/fstree"
+	"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/kvio"
 	meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase"
 	"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode"
 	"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger"
@@ -22,7 +23,6 @@ import (
 	usertest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user/test"
 	versionSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/version"
 	"github.com/stretchr/testify/require"
-	"go.etcd.io/bbolt"
 	"go.uber.org/zap/zaptest"
 )
 
@@ -151,17 +151,15 @@ func TestFlush(t *testing.T) {
 		t.Run("db, invalid address", func(t *testing.T) {
 			testIgnoreErrors(t, func(c *cache) {
 				_, data := newObject(t, 1)
-				require.NoError(t, c.db.Batch(func(tx *bbolt.Tx) error {
+				require.NoError(t, c.repo.Write(func(tx kvio.WriteOnlyTx) error {
-					b := tx.Bucket(defaultBucket)
+					return tx.Write([]byte{1, 2, 3}, data)
-					return b.Put([]byte{1, 2, 3}, data)
 				}))
 			})
 		})
 		t.Run("db, invalid object", func(t *testing.T) {
 			testIgnoreErrors(t, func(c *cache) {
-				require.NoError(t, c.db.Batch(func(tx *bbolt.Tx) error {
+				require.NoError(t, c.repo.Write(func(tx kvio.WriteOnlyTx) error {
-					b := tx.Bucket(defaultBucket)
+					return tx.Write([]byte(oidtest.Address().EncodeToString()), []byte{1, 2, 3})
-					return b.Put([]byte(oidtest.Address().EncodeToString()), []byte{1, 2, 3})
 				}))
 			})
 		})

pkg/local_object_storage/writecache/get.go

@@ -6,13 +6,12 @@ import (
 
 	"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common"
 	"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/metaerr"
+	"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/kvio"
 	"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr"
 	"git.frostfs.info/TrueCloudLab/frostfs-observability/tracing"
 	apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status"
 	objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
 	oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
-	"github.com/nspcc-dev/neo-go/pkg/util/slice"
-	"go.etcd.io/bbolt"
 	"go.opentelemetry.io/otel/attribute"
 	"go.opentelemetry.io/otel/trace"
 )
@@ -41,7 +40,7 @@ func (c *cache) getInternal(ctx context.Context, saddr string, addr oid.Address)
 		c.metrics.Get(time.Since(startedAt), found, storageType)
 	}()
 
-	value, err := Get(c.db, []byte(saddr))
+	value, err := Get(c.repo, []byte(saddr))
 	if err == nil {
 		obj := objectSDK.New()
 		found = true
@@ -83,19 +82,13 @@ func (c *cache) Head(ctx context.Context, addr oid.Address) (*objectSDK.Object,
 // Key should be a stringified address.
 //
 // Returns an error of type apistatus.ObjectNotFound if the requested object is missing in db.
-func Get(db *bbolt.DB, key []byte) ([]byte, error) {
+func Get(repo kvio.Repository, key []byte) ([]byte, error) {
-	var value []byte
+	value, err := kvio.Read(repo, key)
-	err := db.View(func(tx *bbolt.Tx) error {
+	if err != nil {
-		b := tx.Bucket(defaultBucket)
+		return nil, metaerr.Wrap(err)
-		if b == nil {
-			return ErrNoDefaultBucket
 	}
-		value = b.Get(key)
 	if value == nil {
-			return logicerr.Wrap(apistatus.ObjectNotFound{})
+		return nil, metaerr.Wrap(logicerr.Wrap(apistatus.ObjectNotFound{}))
 	}
-		value = slice.Copy(value)
+	return value, nil
-		return nil
-	})
-	return value, metaerr.Wrap(err)
 }

pkg/local_object_storage/writecache/iterate.go

@@ -1,39 +0,0 @@
-package writecache
-
-import (
-	"errors"
-	"fmt"
-
-	"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/metaerr"
-	oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
-	"go.etcd.io/bbolt"
-)
-
-// ErrNoDefaultBucket is returned by IterateDB when default bucket for objects is missing.
-var ErrNoDefaultBucket = errors.New("no default bucket")
-
-// IterateDB iterates over all objects stored in bbolt.DB instance and passes them to f until error return.
-// It is assumed that db is an underlying database of some WriteCache instance.
-//
-// Returns ErrNoDefaultBucket if there is no default bucket in db.
-//
-// DB must not be nil and should be opened.
-func IterateDB(db *bbolt.DB, f func(oid.Address) error) error {
-	return metaerr.Wrap(db.View(func(tx *bbolt.Tx) error {
-		b := tx.Bucket(defaultBucket)
-		if b == nil {
-			return ErrNoDefaultBucket
-		}
-
-		var addr oid.Address
-
-		return b.ForEach(func(k, v []byte) error {
-			err := addr.DecodeString(string(k))
-			if err != nil {
-				return fmt.Errorf("could not parse object address: %w", err)
-			}
-
-			return f(addr)
-		})
-	}))
-}

pkg/local_object_storage/writecache/mode.go

@@ -48,8 +48,8 @@ func (c *cache) setMode(ctx context.Context, m mode.Mode) error {
 		}
 	}
 
-	if c.db != nil {
+	if c.repo != nil {
-		if err = c.db.Close(); err != nil {
+		if err = c.repo.Close(); err != nil {
 			return fmt.Errorf("can't close write-cache database: %w", err)
 		}
 	}

pkg/local_object_storage/writecache/put.go

@@ -8,8 +8,8 @@ import (
 	"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common"
 	storagelog "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/log"
 	"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/metaerr"
+	"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/kvio"
 	"git.frostfs.info/TrueCloudLab/frostfs-observability/tracing"
-	"go.etcd.io/bbolt"
 	"go.opentelemetry.io/otel/attribute"
 	"go.opentelemetry.io/otel/trace"
 )
@@ -84,10 +84,7 @@ func (c *cache) putSmall(obj objectInfo) error {
 		return ErrOutOfSpace
 	}
 
-	err := c.db.Batch(func(tx *bbolt.Tx) error {
+	err := kvio.Write(c.repo, []byte(obj.addr), obj.data)
-		b := tx.Bucket(defaultBucket)
-		return b.Put([]byte(obj.addr), obj.data)
-	})
 	if err == nil {
 		storagelog.Write(c.log,
 			storagelog.AddressField(obj.addr),

pkg/local_object_storage/writecache/state.go

@@ -4,8 +4,6 @@ import (
 	"fmt"
 	"math"
 	"sync/atomic"
-
-	"go.etcd.io/bbolt"
 )
 
 func (c *cache) estimateCacheSize() uint64 {
@@ -52,18 +50,13 @@ func (x *counters) FS() uint64 {
 }
 
 func (c *cache) initCounters() error {
-	var inDB uint64
+	stats, err := c.repo.Stats()
-	err := c.db.View(func(tx *bbolt.Tx) error {
-		b := tx.Bucket(defaultBucket)
-		if b != nil {
-			inDB = uint64(b.Stats().KeyN)
-		}
-		return nil
-	})
 	if err != nil {
 		return fmt.Errorf("could not read write-cache DB counter: %w", err)
 	}
 
+	inDB := stats.KeyCount
+
 	inFS, err := c.fsTree.NumberOfObjects()
 	if err != nil {
 		return fmt.Errorf("could not read write-cache FS counter: %w", err)

pkg/local_object_storage/writecache/storage.go

@@ -5,50 +5,49 @@ import (
 	"errors"
 	"fmt"
 	"os"
+	"path/filepath"
+	"time"
 
 	"git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs"
 	"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common"
 	"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/fstree"
 	storagelog "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/log"
+	"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/kvio"
+	bboltrepo "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/kvio/bbolt"
 	"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util"
 	apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status"
 	oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
+
 	"go.etcd.io/bbolt"
 	"go.uber.org/zap"
 )
 
-// store represents persistent storage with in-memory LRU cache
-// for flushed items on top of it.
-type store struct {
-	db *bbolt.DB
-}
-
-const dbName = "small.bolt"
-
 func (c *cache) openStore(readOnly bool) error {
 	err := util.MkdirAllX(c.path, os.ModePerm)
 	if err != nil {
 		return err
 	}
 
-	c.db, err = OpenDB(c.path, readOnly, c.openFile)
+	// It's possible to use other Repository implementations here, such as the badger-backed one:
+	//
+	// opts := badger.DefaultOptions(filepath.Join(c.path, "small.badger")).WithReadOnly(readOnly)
+	// c.repo, err = badgerrepo.Open(opts)
+
+	c.repo, err = bboltrepo.Open(filepath.Join(c.path, "small.bolt"), &bboltrepo.Options{
+		Options: bbolt.Options{
+			NoFreelistSync: true,
+			ReadOnly:       readOnly,
+			Timeout:        100 * time.Millisecond,
+			OpenFile:       c.openFile,
+		},
+		MaxBatchSize:  c.maxBatchSize,
+		MaxBatchDelay: c.maxBatchDelay,
+	})
+
 	if err != nil {
 		return fmt.Errorf("could not open database: %w", err)
 	}
 
-	c.db.MaxBatchSize = c.maxBatchSize
-	c.db.MaxBatchDelay = c.maxBatchDelay
-
-	if !readOnly {
-		err = c.db.Update(func(tx *bbolt.Tx) error {
-			_, err := tx.CreateBucketIfNotExists(defaultBucket)
-			return err
-		})
-		if err != nil {
-			return fmt.Errorf("could not create default bucket: %w", err)
-		}
-	}
-
 	c.fsTree = fstree.New(
 		fstree.WithPath(c.path),
 		fstree.WithPerm(os.ModePerm),
@@ -68,10 +67,9 @@ func (c *cache) deleteFromDB(keys []string) []string {
 	}
 
 	var errorIndex int
-	err := c.db.Batch(func(tx *bbolt.Tx) error {
+	err := c.repo.Write(func(tx kvio.WriteOnlyTx) error {
-		b := tx.Bucket(defaultBucket)
 		for errorIndex = range keys {
-			if err := b.Delete([]byte(keys[errorIndex])); err != nil {
+			if err := tx.Delete([]byte(keys[errorIndex])); err != nil {
 				return err
 			}
 		}

pkg/local_object_storage/writecache/util.go

@@ -1,20 +0,0 @@
-package writecache
-
-import (
-	"io/fs"
-	"os"
-	"path/filepath"
-	"time"
-
-	"go.etcd.io/bbolt"
-)
-
-// OpenDB opens BoltDB instance for write-cache. Opens in read-only mode if ro is true.
-func OpenDB(p string, ro bool, openFile func(string, int, fs.FileMode) (*os.File, error)) (*bbolt.DB, error) {
-	return bbolt.Open(filepath.Join(p, dbName), os.ModePerm, &bbolt.Options{
-		NoFreelistSync: true,
-		ReadOnly:       ro,
-		Timeout:        100 * time.Millisecond,
-		OpenFile:       openFile,
-	})
-}

pkg/local_object_storage/writecache/writecache.go

@@ -8,6 +8,7 @@ import (
 	"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common"
 	"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/fstree"
 	"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/metaerr"
+	"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/kvio"
 	"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode"
 	"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger"
 	"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
@@ -63,8 +64,8 @@ type cache struct {
 	closeCh chan struct{}
 	// wg is a wait group for flush workers.
 	wg sync.WaitGroup
-	// store contains underlying database.
+	// repo is the underlying key/value repository.
-	store
+	repo kvio.Repository
 	// fsTree contains big files stored directly on file-system.
 	fsTree *fstree.FSTree
 }
@@ -84,10 +85,6 @@ const (
 	defaultMaxCacheSize    = 1 << 30          // 1 GiB
 )
 
-var (
-	defaultBucket = []byte{0}
-)
-
 // New creates new writecache instance.
 func New(opts ...Option) Cache {
 	c := &cache{
@@ -165,10 +162,10 @@ func (c *cache) Close() error {
 
 	c.closeCh = nil
 	var err error
-	if c.db != nil {
+	if c.repo != nil {
-		err = c.db.Close()
+		err = c.repo.Close()
 		if err != nil {
-			c.db = nil
+			c.repo = nil
 		}
 	}
 	c.metrics.Close()