[#421] Try using badger for the write-cache

Signed-off-by: Alejandro Lopez <a.lopez@yadro.com>

pkg/local_object_storage/writecache/writecachebbolt/cachebbolt.go

@@ -0,0 +1,146 @@
+package writecachebbolt
+
+import (
+	"os"
+	"sync"
+
+	"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/fstree"
+	"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/metaerr"
+	"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode"
+	"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/writecache"
+	"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger"
+	objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
+	"go.etcd.io/bbolt"
+	"go.uber.org/zap"
+)
+
+type cache struct {
+	options
+
+	// mtx protects statistics, counters and compressFlags.
+	mtx sync.RWMutex
+
+	mode    mode.Mode
+	modeMtx sync.RWMutex
+
+	// compressFlags maps address of a big object to boolean value indicating
+	// whether object should be compressed.
+	compressFlags map[string]struct{}
+
+	// flushCh is a channel with objects to flush.
+	flushCh chan *objectSDK.Object
+	// closeCh is close channel, protected by modeMtx.
+	closeCh chan struct{}
+	// wg is a wait group for flush workers.
+	wg sync.WaitGroup
+	// store contains underlying database.
+	store
+	// fsTree contains big files stored directly on file-system.
+	fsTree *fstree.FSTree
+}
+
+// wcStorageType is used for write-cache operations logging.
+const wcStorageType = "write-cache"
+
+type objectInfo struct {
+	addr string
+	data []byte
+	obj  *objectSDK.Object
+}
+
+const (
+	defaultMaxObjectSize   = 64 * 1024 * 1024 // 64 MiB
+	defaultSmallObjectSize = 32 * 1024        // 32 KiB
+	defaultMaxCacheSize    = 1 << 30          // 1 GiB
+)
+
+var (
+	defaultBucket = []byte{0}
+)
+
+// New creates new writecache instance.
+func New(opts ...Option) writecache.Cache {
+	c := &cache{
+		flushCh: make(chan *objectSDK.Object),
+		mode:    mode.ReadWrite,
+
+		compressFlags: make(map[string]struct{}),
+		options: options{
+			log:             &logger.Logger{Logger: zap.NewNop()},
+			maxObjectSize:   defaultMaxObjectSize,
+			smallObjectSize: defaultSmallObjectSize,
+			workersCount:    defaultFlushWorkersCount,
+			maxCacheSize:    defaultMaxCacheSize,
+			maxBatchSize:    bbolt.DefaultMaxBatchSize,
+			maxBatchDelay:   bbolt.DefaultMaxBatchDelay,
+			openFile:        os.OpenFile,
+			metrics:         writecache.DefaultMetrics(),
+		},
+	}
+
+	for i := range opts {
+		opts[i](&c.options)
+	}
+
+	return c
+}
+
+// SetLogger sets logger. It is used after the shard ID was generated to use it in logs.
+func (c *cache) SetLogger(l *logger.Logger) {
+	c.log = l
+}
+
+func (c *cache) DumpInfo() writecache.Info {
+	return writecache.Info{
+		Path: c.path,
+	}
+}
+
+// Open opens and initializes database. Reads object counters from the ObjectCounters instance.
+func (c *cache) Open(readOnly bool) error {
+	err := c.openStore(readOnly)
+	if err != nil {
+		return metaerr.Wrap(err)
+	}
+
+	// Opening after Close is done during maintenance mode,
+	// thus we need to create a channel here.
+	c.closeCh = make(chan struct{})
+
+	return metaerr.Wrap(c.initCounters())
+}
+
+// Init runs necessary services.
+func (c *cache) Init() error {
+	c.metrics.SetMode(c.mode)
+	c.runFlushLoop()
+	return nil
+}
+
+// Close closes db connection and stops services. Executes ObjectCounters.FlushAndClose op.
+func (c *cache) Close() error {
+	// We cannot lock mutex for the whole operation duration
+	// because it is taken by some background workers, so `wg.Wait()` is done without modeMtx.
+	c.modeMtx.Lock()
+	if c.closeCh != nil {
+		close(c.closeCh)
+	}
+	c.mode = mode.DegradedReadOnly // prevent new operations from being processed
+	c.modeMtx.Unlock()
+
+	c.wg.Wait()
+
+	c.modeMtx.Lock()
+	defer c.modeMtx.Unlock()
+
+	c.closeCh = nil
+	var err error
+	if c.db != nil {
+		err = c.db.Close()
+		if err != nil {
+			c.db = nil
+		}
+	}
+	c.metrics.Close()
+	return nil
+}

pkg/local_object_storage/writecache/writecachebbolt/delete.go

@@ -0,0 +1,83 @@
+package writecachebbolt
+
+import (
+	"context"
+	"time"
+
+	"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common"
+	storagelog "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/log"
+	"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/metaerr"
+	"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/writecache"
+	"git.frostfs.info/TrueCloudLab/frostfs-observability/tracing"
+	oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
+	"go.etcd.io/bbolt"
+	"go.opentelemetry.io/otel/attribute"
+	"go.opentelemetry.io/otel/trace"
+)
+
+// Delete removes object from write-cache.
+//
+// Returns an error of type apistatus.ObjectNotFound if object is missing in write-cache.
+func (c *cache) Delete(ctx context.Context, addr oid.Address) error {
+	ctx, span := tracing.StartSpanFromContext(ctx, "writecache.Delete",
+		trace.WithAttributes(
+			attribute.String("address", addr.EncodeToString()),
+		))
+	defer span.End()
+
+	deleted := false
+	storageType := writecache.StorageTypeUndefined
+	startedAt := time.Now()
+	defer func() {
+		c.metrics.Delete(time.Since(startedAt), deleted, storageType)
+	}()
+
+	c.modeMtx.RLock()
+	defer c.modeMtx.RUnlock()
+	if c.readOnly() {
+		return writecache.ErrReadOnly
+	}
+
+	saddr := addr.EncodeToString()
+
+	var dataSize int
+	_ = c.db.View(func(tx *bbolt.Tx) error {
+		b := tx.Bucket(defaultBucket)
+		dataSize = len(b.Get([]byte(saddr)))
+		return nil
+	})
+
+	if dataSize > 0 {
+		storageType = writecache.StorageTypeDB
+		err := c.db.Update(func(tx *bbolt.Tx) error {
+			b := tx.Bucket(defaultBucket)
+			err := b.Delete([]byte(saddr))
+			return err
+		})
+		if err != nil {
+			return err
+		}
+		storagelog.Write(c.log,
+			storagelog.AddressField(saddr),
+			storagelog.StorageTypeField(wcStorageType),
+			storagelog.OpField("db DELETE"),
+		)
+		deleted = true
+		c.objCounters.DecDB()
+		return nil
+	}
+
+	storageType = writecache.StorageTypeFSTree
+	_, err := c.fsTree.Delete(ctx, common.DeletePrm{Address: addr})
+	if err == nil {
+		storagelog.Write(c.log,
+			storagelog.AddressField(saddr),
+			storagelog.StorageTypeField(wcStorageType),
+			storagelog.OpField("fstree DELETE"),
+		)
+		c.objCounters.DecFS()
+		deleted = true
+	}
+
+	return metaerr.Wrap(err)
+}

pkg/local_object_storage/writecache/writecachebbolt/flush.go

@@ -0,0 +1,340 @@
+package writecachebbolt
+
+import (
+	"bytes"
+	"context"
+	"errors"
+	"time"
+
+	"git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs"
+	objectCore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object"
+	"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor"
+	"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common"
+	"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/metaerr"
+	meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase"
+	"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/writecache"
+	"git.frostfs.info/TrueCloudLab/frostfs-observability/tracing"
+	objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
+	oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
+	"github.com/mr-tron/base58"
+	"github.com/nspcc-dev/neo-go/pkg/util/slice"
+	"go.etcd.io/bbolt"
+	"go.opentelemetry.io/otel/attribute"
+	"go.opentelemetry.io/otel/trace"
+	"go.uber.org/zap"
+)
+
+const (
+	// flushBatchSize is amount of keys which will be read from cache to be flushed
+	// to the main storage. It is used to reduce contention between cache put
+	// and cache persist.
+	flushBatchSize = 512
+	// defaultFlushWorkersCount is number of workers for putting objects in main storage.
+	defaultFlushWorkersCount = 20
+	// defaultFlushInterval is default time interval between successive flushes.
+	defaultFlushInterval = time.Second
+)
+
+// runFlushLoop starts background workers which periodically flush objects to the blobstor.
+func (c *cache) runFlushLoop() {
+	ctx, cancel := context.WithCancel(context.Background())
+
+	ch := c.closeCh
+	c.wg.Add(1)
+	go func() {
+		<-ch
+		cancel()
+		c.wg.Done()
+	}()
+
+	for i := 0; i < c.workersCount; i++ {
+		c.wg.Add(1)
+		go c.workerFlushSmall()
+	}
+
+	c.wg.Add(1)
+	go func() {
+		c.workerFlushBig(ctx)
+		c.wg.Done()
+	}()
+
+	c.wg.Add(1)
+	go func() {
+		defer c.wg.Done()
+
+		tt := time.NewTimer(defaultFlushInterval)
+		defer tt.Stop()
+
+		for {
+			select {
+			case <-tt.C:
+				c.flushSmallObjects()
+				tt.Reset(defaultFlushInterval)
+			case <-c.closeCh:
+				return
+			}
+		}
+	}()
+}
+
+func (c *cache) flushSmallObjects() {
+	var lastKey []byte
+	var m []objectInfo
+	for {
+		select {
+		case <-c.closeCh:
+			return
+		default:
+		}
+
+		m = m[:0]
+
+		c.modeMtx.RLock()
+		if c.readOnly() {
+			c.modeMtx.RUnlock()
+			time.Sleep(time.Second)
+			continue
+		}
+
+		// We put objects in batches of fixed size to not interfere with main put cycle a lot.
+		_ = c.db.View(func(tx *bbolt.Tx) error {
+			b := tx.Bucket(defaultBucket)
+			cs := b.Cursor()
+
+			var k, v []byte
+
+			if len(lastKey) == 0 {
+				k, v = cs.First()
+			} else {
+				k, v = cs.Seek(lastKey)
+				if bytes.Equal(k, lastKey) {
+					k, v = cs.Next()
+				}
+			}
+
+			for ; k != nil && len(m) < flushBatchSize; k, v = cs.Next() {
+				if len(lastKey) == len(k) {
+					copy(lastKey, k)
+				} else {
+					lastKey = slice.Copy(k)
+				}
+
+				m = append(m, objectInfo{
+					addr: string(k),
+					data: slice.Copy(v),
+				})
+			}
+			return nil
+		})
+
+		var count int
+		for i := range m {
+			obj := objectSDK.New()
+			if err := obj.Unmarshal(m[i].data); err != nil {
+				continue
+			}
+
+			count++
+			select {
+			case c.flushCh <- obj:
+			case <-c.closeCh:
+				c.modeMtx.RUnlock()
+				return
+			}
+		}
+
+		if count == 0 {
+			c.modeMtx.RUnlock()
+			break
+		}
+
+		c.modeMtx.RUnlock()
+
+		c.log.Debug(logs.WritecacheTriedToFlushItemsFromWritecache,
+			zap.Int("count", count),
+			zap.String("start", base58.Encode(lastKey)))
+	}
+}
+
+func (c *cache) workerFlushBig(ctx context.Context) {
+	tick := time.NewTicker(defaultFlushInterval * 10)
+	for {
+		select {
+		case <-tick.C:
+			c.modeMtx.RLock()
+			if c.readOnly() {
+				c.modeMtx.RUnlock()
+				break
+			}
+
+			_ = c.flushFSTree(ctx, true)
+
+			c.modeMtx.RUnlock()
+		case <-c.closeCh:
+			return
+		}
+	}
+}
+
+func (c *cache) reportFlushError(msg string, addr string, err error) {
+	if c.reportError != nil {
+		c.reportError(msg, err)
+	} else {
+		c.log.Error(msg,
+			zap.String("address", addr),
+			zap.Error(err))
+	}
+}
+
+func (c *cache) flushFSTree(ctx context.Context, ignoreErrors bool) error {
+	var prm common.IteratePrm
+	prm.IgnoreErrors = ignoreErrors
+	prm.LazyHandler = func(addr oid.Address, f func() ([]byte, error)) error {
+		sAddr := addr.EncodeToString()
+
+		data, err := f()
+		if err != nil {
+			c.reportFlushError("can't read a file", sAddr, metaerr.Wrap(err))
+			if ignoreErrors {
+				return nil
+			}
+			return err
+		}
+
+		var obj objectSDK.Object
+		err = obj.Unmarshal(data)
+		if err != nil {
+			c.reportFlushError("can't unmarshal an object", sAddr, metaerr.Wrap(err))
+			if ignoreErrors {
+				return nil
+			}
+			return err
+		}
+
+		err = c.flushObject(ctx, &obj, data, writecache.StorageTypeFSTree)
+		if err != nil {
+			if ignoreErrors {
+				return nil
+			}
+			return err
+		}
+
+		c.deleteFromDisk(ctx, []string{sAddr})
+		return nil
+	}
+
+	_, err := c.fsTree.Iterate(ctx, prm)
+	return err
+}
+
+// workerFlushSmall writes small objects to the main storage.
+func (c *cache) workerFlushSmall() {
+	defer c.wg.Done()
+
+	var obj *objectSDK.Object
+	for {
+		// Give priority to direct put.
+		select {
+		case obj = <-c.flushCh:
+		case <-c.closeCh:
+			return
+		}
+
+		err := c.flushObject(context.TODO(), obj, nil, writecache.StorageTypeDB)
+		if err != nil {
+			// Error is handled in flushObject.
+			continue
+		}
+
+		c.deleteFromDB([]string{objectCore.AddressOf(obj).EncodeToString()})
+	}
+}
+
+// flushObject is used to write object directly to the main storage.
+func (c *cache) flushObject(ctx context.Context, obj *objectSDK.Object, data []byte, st writecache.StorageType) error {
+	var err error
+
+	defer func() {
+		c.metrics.Flush(err == nil, st)
+	}()
+
+	addr := objectCore.AddressOf(obj)
+
+	var prm common.PutPrm
+	prm.Object = obj
+	prm.RawData = data
+
+	res, err := c.blobstor.Put(ctx, prm)
+	if err != nil {
+		if !errors.Is(err, common.ErrNoSpace) && !errors.Is(err, common.ErrReadOnly) &&
+			!errors.Is(err, blobstor.ErrNoPlaceFound) {
+			c.reportFlushError("can't flush an object to blobstor",
+				addr.EncodeToString(), err)
+		}
+		return err
+	}
+
+	var updPrm meta.UpdateStorageIDPrm
+	updPrm.SetAddress(addr)
+	updPrm.SetStorageID(res.StorageID)
+
+	_, err = c.metabase.UpdateStorageID(updPrm)
+	if err != nil {
+		c.reportFlushError("can't update object storage ID",
+			addr.EncodeToString(), err)
+	}
+	return err
+}
+
+// Flush flushes all objects from the write-cache to the main storage.
+// Write-cache must be in readonly mode to ensure correctness of an operation and
+// to prevent interference with background flush workers.
+func (c *cache) Flush(ctx context.Context, ignoreErrors bool) error {
+	ctx, span := tracing.StartSpanFromContext(ctx, "writecache.Flush",
+		trace.WithAttributes(
+			attribute.Bool("ignore_errors", ignoreErrors),
+		))
+	defer span.End()
+
+	c.modeMtx.RLock()
+	defer c.modeMtx.RUnlock()
+
+	return c.flush(ctx, ignoreErrors)
+}
+
+func (c *cache) flush(ctx context.Context, ignoreErrors bool) error {
+	if err := c.flushFSTree(ctx, ignoreErrors); err != nil {
+		return err
+	}
+
+	return c.db.View(func(tx *bbolt.Tx) error {
+		var addr oid.Address
+
+		b := tx.Bucket(defaultBucket)
+		cs := b.Cursor()
+		for k, data := cs.Seek(nil); k != nil; k, data = cs.Next() {
+			sa := string(k)
+			if err := addr.DecodeString(sa); err != nil {
+				c.reportFlushError("can't decode object address from the DB", sa, metaerr.Wrap(err))
+				if ignoreErrors {
+					continue
+				}
+				return err
+			}
+
+			var obj objectSDK.Object
+			if err := obj.Unmarshal(data); err != nil {
+				c.reportFlushError("can't unmarshal an object from the DB", sa, metaerr.Wrap(err))
+				if ignoreErrors {
+					continue
+				}
+				return err
+			}
+
+			if err := c.flushObject(ctx, &obj, data, writecache.StorageTypeDB); err != nil {
+				return err
+			}
+		}
+		return nil
+	})
+}

pkg/local_object_storage/writecache/writecachebbolt/flush_test.go

@@ -0,0 +1,106 @@
+package writecachebbolt
+
+import (
+	"context"
+	"os"
+	"path/filepath"
+	"sync/atomic"
+	"testing"
+
+	objectCore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object"
+	"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor"
+	"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common"
+	"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/testutil"
+	meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase"
+	"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/writecache"
+	"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/writecache/writecachetest"
+	"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger"
+	oidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id/test"
+	"github.com/stretchr/testify/require"
+	"go.etcd.io/bbolt"
+	"go.uber.org/zap/zaptest"
+)
+
+func TestFlush(t *testing.T) {
+	createCacheFn := func(t *testing.T, smallSize uint64, mb *meta.DB, bs *blobstor.BlobStor, opts ...Option) writecache.Cache {
+		return New(
+			append([]Option{
+				WithLogger(&logger.Logger{Logger: zaptest.NewLogger(t)}),
+				WithPath(filepath.Join(t.TempDir(), "writecache")),
+				WithSmallObjectSize(smallSize),
+				WithMetabase(mb),
+				WithBlobstor(bs),
+			}, opts...)...)
+	}
+
+	errCountOpt := func() (Option, *atomic.Uint32) {
+		cnt := &atomic.Uint32{}
+		return WithReportErrorFunc(func(string, error) {
+			cnt.Add(1)
+		}), cnt
+	}
+
+	failures := []writecachetest.TestFailureInjector[Option]{
+		{
+			Desc: "db, invalid address",
+			InjectFn: func(t *testing.T, wc writecache.Cache) {
+				c := wc.(*cache)
+				obj := testutil.GenerateObject()
+				data, err := obj.Marshal()
+				require.NoError(t, err)
+				require.NoError(t, c.db.Batch(func(tx *bbolt.Tx) error {
+					b := tx.Bucket(defaultBucket)
+					return b.Put([]byte{1, 2, 3}, data)
+				}))
+			},
+		},
+		{
+			Desc: "db, invalid object",
+			InjectFn: func(t *testing.T, wc writecache.Cache) {
+				c := wc.(*cache)
+				require.NoError(t, c.db.Batch(func(tx *bbolt.Tx) error {
+					b := tx.Bucket(defaultBucket)
+					k := []byte(oidtest.Address().EncodeToString())
+					v := []byte{1, 2, 3}
+					return b.Put(k, v)
+				}))
+			},
+		},
+		{
+			Desc: "fs, read error",
+			InjectFn: func(t *testing.T, wc writecache.Cache) {
+				c := wc.(*cache)
+				obj := testutil.GenerateObject()
+				data, err := obj.Marshal()
+				require.NoError(t, err)
+
+				var prm common.PutPrm
+				prm.Address = objectCore.AddressOf(obj)
+				prm.RawData = data
+
+				_, err = c.fsTree.Put(context.Background(), prm)
+				require.NoError(t, err)
+
+				p := prm.Address.Object().EncodeToString() + "." + prm.Address.Container().EncodeToString()
+				p = filepath.Join(c.fsTree.RootPath, p[:1], p[1:])
+
+				_, err = os.Stat(p) // sanity check
+				require.NoError(t, err)
+				require.NoError(t, os.Truncate(p, 0)) // corrupt the file contents, so that it can't be unmarshalled
+			},
+		},
+		{
+			Desc: "fs, invalid object",
+			InjectFn: func(t *testing.T, wc writecache.Cache) {
+				c := wc.(*cache)
+				var prm common.PutPrm
+				prm.Address = oidtest.Address()
+				prm.RawData = []byte{1, 2, 3}
+				_, err := c.fsTree.Put(context.Background(), prm)
+				require.NoError(t, err)
+			},
+		},
+	}
+
+	writecachetest.TestFlush(t, createCacheFn, errCountOpt, failures...)
+}

pkg/local_object_storage/writecache/writecachebbolt/generic_test.go

@@ -0,0 +1,18 @@
+package writecachebbolt
+
+import (
+	"testing"
+
+	"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/storagetest"
+	"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger"
+	"go.uber.org/zap/zaptest"
+)
+
+func TestGeneric(t *testing.T) {
+	storagetest.TestAll(t, func(t *testing.T) storagetest.Component {
+		return New(
+			WithLogger(&logger.Logger{Logger: zaptest.NewLogger(t)}),
+			WithFlushWorkersCount(2),
+			WithPath(t.TempDir()))
+	})
+}

pkg/local_object_storage/writecache/writecachebbolt/get.go

@@ -0,0 +1,102 @@
+package writecachebbolt
+
+import (
+	"context"
+	"time"
+
+	"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common"
+	"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/metaerr"
+	"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr"
+	"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/writecache"
+	"git.frostfs.info/TrueCloudLab/frostfs-observability/tracing"
+	apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status"
+	objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
+	oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
+	"github.com/nspcc-dev/neo-go/pkg/util/slice"
+	"go.etcd.io/bbolt"
+	"go.opentelemetry.io/otel/attribute"
+	"go.opentelemetry.io/otel/trace"
+)
+
+// Get returns object from write-cache.
+//
+// Returns an error of type apistatus.ObjectNotFound if the requested object is missing in write-cache.
+func (c *cache) Get(ctx context.Context, addr oid.Address) (*objectSDK.Object, error) {
+	saddr := addr.EncodeToString()
+
+	ctx, span := tracing.StartSpanFromContext(ctx, "writecache.Get",
+		trace.WithAttributes(
+			attribute.String("address", saddr),
+		))
+	defer span.End()
+
+	obj, err := c.getInternal(ctx, saddr, addr)
+	return obj, metaerr.Wrap(err)
+}
+
+func (c *cache) getInternal(ctx context.Context, saddr string, addr oid.Address) (*objectSDK.Object, error) {
+	found := false
+	storageType := writecache.StorageTypeUndefined
+	startedAt := time.Now()
+	defer func() {
+		c.metrics.Get(time.Since(startedAt), found, storageType)
+	}()
+
+	value, err := Get(c.db, []byte(saddr))
+	if err == nil {
+		obj := objectSDK.New()
+		found = true
+		storageType = writecache.StorageTypeDB
+		return obj, obj.Unmarshal(value)
+	}
+
+	res, err := c.fsTree.Get(ctx, common.GetPrm{Address: addr})
+	if err != nil {
+		return nil, logicerr.Wrap(apistatus.ObjectNotFound{})
+	}
+
+	found = true
+	storageType = writecache.StorageTypeFSTree
+	return res.Object, nil
+}
+
+// Head returns object header from write-cache.
+//
+// Returns an error of type apistatus.ObjectNotFound if the requested object is missing in write-cache.
+func (c *cache) Head(ctx context.Context, addr oid.Address) (*objectSDK.Object, error) {
+	saddr := addr.EncodeToString()
+
+	ctx, span := tracing.StartSpanFromContext(ctx, "writecache.Head",
+		trace.WithAttributes(
+			attribute.String("address", saddr),
+		))
+	defer span.End()
+
+	obj, err := c.getInternal(ctx, saddr, addr)
+	if err != nil {
+		return nil, metaerr.Wrap(err)
+	}
+
+	return obj.CutPayload(), nil
+}
+
+// Get fetches object from the underlying database.
+// Key should be a stringified address.
+//
+// Returns an error of type apistatus.ObjectNotFound if the requested object is missing in db.
+func Get(db *bbolt.DB, key []byte) ([]byte, error) {
+	var value []byte
+	err := db.View(func(tx *bbolt.Tx) error {
+		b := tx.Bucket(defaultBucket)
+		if b == nil {
+			return ErrNoDefaultBucket
+		}
+		value = b.Get(key)
+		if value == nil {
+			return logicerr.Wrap(apistatus.ObjectNotFound{})
+		}
+		value = slice.Copy(value)
+		return nil
+	})
+	return value, metaerr.Wrap(err)
+}

pkg/local_object_storage/writecache/writecachebbolt/iterate.go

@@ -0,0 +1,39 @@
+package writecachebbolt
+
+import (
+	"errors"
+	"fmt"
+
+	"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/metaerr"
+	oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
+	"go.etcd.io/bbolt"
+)
+
+// ErrNoDefaultBucket is returned by IterateDB when default bucket for objects is missing.
+var ErrNoDefaultBucket = errors.New("no default bucket")
+
+// IterateDB iterates over all objects stored in bbolt.DB instance and passes them to f until error return.
+// It is assumed that db is an underlying database of some WriteCache instance.
+//
+// Returns ErrNoDefaultBucket if there is no default bucket in db.
+//
+// DB must not be nil and should be opened.
+func IterateDB(db *bbolt.DB, f func(oid.Address) error) error {
+	return metaerr.Wrap(db.View(func(tx *bbolt.Tx) error {
+		b := tx.Bucket(defaultBucket)
+		if b == nil {
+			return ErrNoDefaultBucket
+		}
+
+		var addr oid.Address
+
+		return b.ForEach(func(k, v []byte) error {
+			err := addr.DecodeString(string(k))
+			if err != nil {
+				return fmt.Errorf("could not parse object address: %w", err)
+			}
+
+			return f(addr)
+		})
+	}))
+}

pkg/local_object_storage/writecache/writecachebbolt/mode.go

@@ -0,0 +1,75 @@
+package writecachebbolt
+
+import (
+	"context"
+	"fmt"
+	"time"
+
+	"git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs"
+	"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode"
+	"git.frostfs.info/TrueCloudLab/frostfs-observability/tracing"
+	"go.opentelemetry.io/otel/attribute"
+	"go.opentelemetry.io/otel/trace"
+)
+
+// SetMode sets write-cache mode of operation.
+// When shard is put in read-only mode all objects in memory are flushed to disk
+// and all background jobs are suspended.
+func (c *cache) SetMode(m mode.Mode) error {
+	ctx, span := tracing.StartSpanFromContext(context.TODO(), "writecache.SetMode",
+		trace.WithAttributes(
+			attribute.String("mode", m.String()),
+		))
+	defer span.End()
+
+	err := c.setMode(ctx, m)
+	if err == nil {
+		c.metrics.SetMode(m)
+	}
+	return err
+}
+
+// setMode applies new mode. Must be called with cache.modeMtx lock taken.
+func (c *cache) setMode(ctx context.Context, m mode.Mode) error {
+	var err error
+	turnOffMeta := m.NoMetabase()
+
+	if turnOffMeta && !c.mode.NoMetabase() {
+		err = c.flush(ctx, true)
+		if err != nil {
+			return err
+		}
+	}
+
+	if c.db != nil {
+		if err = c.db.Close(); err != nil {
+			return fmt.Errorf("can't close write-cache database: %w", err)
+		}
+	}
+
+	// Suspend producers to ensure there are channel send operations in fly.
+	// flushCh is populated by `flush` with `modeMtx` taken, thus waiting until it is empty
+	// guarantees that there are no in-fly operations.
+	for len(c.flushCh) != 0 {
+		c.log.Info(logs.WritecacheWaitingForChannelsToFlush)
+		time.Sleep(time.Second)
+	}
+
+	if turnOffMeta {
+		c.mode = m
+		return nil
+	}
+
+	if err = c.openStore(m.ReadOnly()); err != nil {
+		return err
+	}
+
+	c.mode = m
+	return nil
+}
+
+// readOnly returns true if current mode is read-only.
+// `c.modeMtx` must be taken.
+func (c *cache) readOnly() bool {
+	return c.mode.ReadOnly()
+}

pkg/local_object_storage/writecache/writecachebbolt/options.go

@@ -0,0 +1,176 @@
+package writecachebbolt
+
+import (
+	"context"
+	"io/fs"
+	"os"
+	"time"
+
+	"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor"
+	"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common"
+	meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase"
+	"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/writecache"
+	"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger"
+	objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
+	"go.uber.org/zap"
+)
+
+// Option represents write-cache configuration option.
+type Option func(*options)
+
+// meta is an interface for a metabase.
+type metabase interface {
+	Exists(context.Context, meta.ExistsPrm) (meta.ExistsRes, error)
+	StorageID(context.Context, meta.StorageIDPrm) (meta.StorageIDRes, error)
+	UpdateStorageID(meta.UpdateStorageIDPrm) (meta.UpdateStorageIDRes, error)
+}
+
+// blob is an interface for the blobstor.
+type blob interface {
+	Put(context.Context, common.PutPrm) (common.PutRes, error)
+	NeedsCompression(obj *objectSDK.Object) bool
+	Exists(ctx context.Context, res common.ExistsPrm) (common.ExistsRes, error)
+}
+
+type options struct {
+	log *logger.Logger
+	// path is a path to a directory for write-cache.
+	path string
+	// blobstor is the main persistent storage.
+	blobstor blob
+	// metabase is the metabase instance.
+	metabase metabase
+	// maxObjectSize is the maximum size of the object stored in the write-cache.
+	maxObjectSize uint64
+	// smallObjectSize is the maximum size of the object stored in the database.
+	smallObjectSize uint64
+	// workersCount is the number of workers flushing objects in parallel.
+	workersCount int
+	// maxCacheSize is the maximum total size of all objects saved in cache (DB + FS).
+	// 1 GiB by default.
+	maxCacheSize uint64
+	// objCounters contains atomic counters for the number of objects stored in cache.
+	objCounters counters
+	// maxBatchSize is the maximum batch size for the small object database.
+	maxBatchSize int
+	// maxBatchDelay is the maximum batch wait time for the small object database.
+	maxBatchDelay time.Duration
+	// noSync is true iff FSTree allows unsynchronized writes.
+	noSync bool
+	// reportError is the function called when encountering disk errors in background workers.
+	reportError func(string, error)
+	// openFile is the function called internally by bbolt to open database files. Useful for hermetic testing.
+	openFile func(string, int, fs.FileMode) (*os.File, error)
+	// metrics is metrics implementation
+	metrics writecache.Metrics
+}
+
+// WithLogger sets logger.
+func WithLogger(log *logger.Logger) Option {
+	return func(o *options) {
+		o.log = &logger.Logger{Logger: log.With(zap.String("component", "WriteCache"))}
+	}
+}
+
+// WithPath sets path to writecache db.
+func WithPath(path string) Option {
+	return func(o *options) {
+		o.path = path
+	}
+}
+
+// WithBlobstor sets main object storage.
+func WithBlobstor(bs *blobstor.BlobStor) Option {
+	return func(o *options) {
+		o.blobstor = bs
+	}
+}
+
+// WithMetabase sets metabase.
+func WithMetabase(db *meta.DB) Option {
+	return func(o *options) {
+		o.metabase = db
+	}
+}
+
+// WithMaxObjectSize sets maximum object size to be stored in write-cache.
+func WithMaxObjectSize(sz uint64) Option {
+	return func(o *options) {
+		if sz > 0 {
+			o.maxObjectSize = sz
+		}
+	}
+}
+
+// WithSmallObjectSize sets maximum object size to be stored in write-cache.
+func WithSmallObjectSize(sz uint64) Option {
+	return func(o *options) {
+		if sz > 0 {
+			o.smallObjectSize = sz
+		}
+	}
+}
+
+func WithFlushWorkersCount(c int) Option {
+	return func(o *options) {
+		if c > 0 {
+			o.workersCount = c
+		}
+	}
+}
+
+// WithMaxCacheSize sets maximum write-cache size in bytes.
+func WithMaxCacheSize(sz uint64) Option {
+	return func(o *options) {
+		o.maxCacheSize = sz
+	}
+}
+
+// WithMaxBatchSize sets max batch size for the small object database.
+func WithMaxBatchSize(sz int) Option {
+	return func(o *options) {
+		if sz > 0 {
+			o.maxBatchSize = sz
+		}
+	}
+}
+
+// WithMaxBatchDelay sets max batch delay for the small object database.
+func WithMaxBatchDelay(d time.Duration) Option {
+	return func(o *options) {
+		if d > 0 {
+			o.maxBatchDelay = d
+		}
+	}
+}
+
+// WithNoSync sets an option to allow returning to caller on PUT before write is persisted.
+// Note, that we use this flag for FSTree only and DO NOT use it for a bolt DB because
+// we cannot yet properly handle the corrupted database during the startup. This SHOULD NOT
+// be relied upon and may be changed in future.
+func WithNoSync(noSync bool) Option {
+	return func(o *options) {
+		o.noSync = noSync
+	}
+}
+
+// WithReportErrorFunc sets error reporting function.
+func WithReportErrorFunc(f func(string, error)) Option {
+	return func(o *options) {
+		o.reportError = f
+	}
+}
+
+// WithOpenFile sets the OpenFile function to use internally by bolt. Useful for hermetic testing.
+func WithOpenFile(f func(string, int, fs.FileMode) (*os.File, error)) Option {
+	return func(o *options) {
+		o.openFile = f
+	}
+}
+
+// WithMetrics sets metrics implementation.
+func WithMetrics(metrics writecache.Metrics) Option {
+	return func(o *options) {
+		o.metrics = metrics
+	}
+}

pkg/local_object_storage/writecache/writecachebbolt/put.go

@@ -0,0 +1,128 @@
+package writecachebbolt
+
+import (
+	"context"
+	"errors"
+	"time"
+
+	"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common"
+	storagelog "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/log"
+	"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/metaerr"
+	"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/writecache"
+	"git.frostfs.info/TrueCloudLab/frostfs-observability/tracing"
+	"go.etcd.io/bbolt"
+	"go.opentelemetry.io/otel/attribute"
+	"go.opentelemetry.io/otel/trace"
+)
+
+var (
+	// ErrBigObject is returned when object is too big to be placed in cache.
+	ErrBigObject = errors.New("too big object")
+	// ErrOutOfSpace is returned when there is no space left to put a new object.
+	ErrOutOfSpace = errors.New("no space left in the write cache")
+)
+
+// Put puts object to write-cache.
+//
+// Returns ErrReadOnly if write-cache is in R/O mode.
+// Returns ErrNotInitialized if write-cache has not been initialized yet.
+// Returns ErrOutOfSpace if saving an object leads to WC's size overflow.
+// Returns ErrBigObject if an objects exceeds maximum object size.
+func (c *cache) Put(ctx context.Context, prm common.PutPrm) (common.PutRes, error) {
+	ctx, span := tracing.StartSpanFromContext(ctx, "writecache.Put",
+		trace.WithAttributes(
+			attribute.String("address", prm.Address.EncodeToString()),
+			attribute.Bool("dont_compress", prm.DontCompress),
+		))
+	defer span.End()
+
+	startedAt := time.Now()
+	added := false
+	storageType := writecache.StorageTypeUndefined
+	defer func() {
+		c.metrics.Put(time.Since(startedAt), added, storageType)
+	}()
+
+	c.modeMtx.RLock()
+	defer c.modeMtx.RUnlock()
+	if c.readOnly() {
+		return common.PutRes{}, writecache.ErrReadOnly
+	}
+
+	sz := uint64(len(prm.RawData))
+	if sz > c.maxObjectSize {
+		return common.PutRes{}, ErrBigObject
+	}
+
+	oi := objectInfo{
+		addr: prm.Address.EncodeToString(),
+		obj:  prm.Object,
+		data: prm.RawData,
+	}
+
+	if sz <= c.smallObjectSize {
+		storageType = writecache.StorageTypeDB
+		err := c.putSmall(oi)
+		if err == nil {
+			added = true
+		}
+		return common.PutRes{}, err
+	}
+
+	storageType = writecache.StorageTypeFSTree
+	err := c.putBig(ctx, oi.addr, prm)
+	if err == nil {
+		added = true
+	}
+	return common.PutRes{}, metaerr.Wrap(err)
+}
+
+// putSmall persists small objects to the write-cache database and
+// pushes the to the flush workers queue.
+func (c *cache) putSmall(obj objectInfo) error {
+	cacheSize := c.estimateCacheSize()
+	if c.maxCacheSize < c.incSizeDB(cacheSize) {
+		return ErrOutOfSpace
+	}
+
+	err := c.db.Batch(func(tx *bbolt.Tx) error {
+		b := tx.Bucket(defaultBucket)
+		return b.Put([]byte(obj.addr), obj.data)
+	})
+	if err == nil {
+		storagelog.Write(c.log,
+			storagelog.AddressField(obj.addr),
+			storagelog.StorageTypeField(wcStorageType),
+			storagelog.OpField("db PUT"),
+		)
+		c.objCounters.IncDB()
+	}
+	return err
+}
+
+// putBig writes object to FSTree and pushes it to the flush workers queue.
+func (c *cache) putBig(ctx context.Context, addr string, prm common.PutPrm) error {
+	cacheSz := c.estimateCacheSize()
+	if c.maxCacheSize < c.incSizeFS(cacheSz) {
+		return ErrOutOfSpace
+	}
+
+	_, err := c.fsTree.Put(ctx, prm)
+	if err != nil {
+		return err
+	}
+
+	if c.blobstor.NeedsCompression(prm.Object) {
+		c.mtx.Lock()
+		c.compressFlags[addr] = struct{}{}
+		c.mtx.Unlock()
+	}
+	c.objCounters.IncFS()
+	storagelog.Write(c.log,
+		storagelog.AddressField(addr),
+		storagelog.StorageTypeField(wcStorageType),
+		storagelog.OpField("fstree PUT"),
+	)
+
+	return nil
+}

pkg/local_object_storage/writecache/writecachebbolt/state.go

@@ -0,0 +1,77 @@
+package writecachebbolt
+
+import (
+	"fmt"
+	"math"
+	"sync/atomic"
+
+	"go.etcd.io/bbolt"
+)
+
+func (c *cache) estimateCacheSize() uint64 {
+	db := c.objCounters.DB() * c.smallObjectSize
+	fstree := c.objCounters.FS() * c.maxObjectSize
+	c.metrics.SetEstimateSize(db, fstree)
+	return db + fstree
+}
+
+func (c *cache) incSizeDB(sz uint64) uint64 {
+	return sz + c.smallObjectSize
+}
+
+func (c *cache) incSizeFS(sz uint64) uint64 {
+	return sz + c.maxObjectSize
+}
+
+type counters struct {
+	cDB, cFS atomic.Uint64
+}
+
+func (x *counters) IncDB() {
+	x.cDB.Add(1)
+}
+
+func (x *counters) DecDB() {
+	x.cDB.Add(math.MaxUint64)
+}
+
+func (x *counters) DB() uint64 {
+	return x.cDB.Load()
+}
+
+func (x *counters) IncFS() {
+	x.cFS.Add(1)
+}
+
+func (x *counters) DecFS() {
+	x.cFS.Add(math.MaxUint64)
+}
+
+func (x *counters) FS() uint64 {
+	return x.cFS.Load()
+}
+
+func (c *cache) initCounters() error {
+	var inDB uint64
+	err := c.db.View(func(tx *bbolt.Tx) error {
+		b := tx.Bucket(defaultBucket)
+		if b != nil {
+			inDB = uint64(b.Stats().KeyN)
+		}
+		return nil
+	})
+	if err != nil {
+		return fmt.Errorf("could not read write-cache DB counter: %w", err)
+	}
+
+	inFS, err := c.fsTree.NumberOfObjects()
+	if err != nil {
+		return fmt.Errorf("could not read write-cache FS counter: %w", err)
+	}
+
+	c.objCounters.cDB.Store(inDB)
+	c.objCounters.cFS.Store(inFS)
+	c.metrics.SetActualCounters(inDB, inFS)
+
+	return nil
+}

pkg/local_object_storage/writecache/writecachebbolt/storage.go

@@ -0,0 +1,132 @@
+package writecachebbolt
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"os"
+
+	"git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs"
+	"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common"
+	"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/fstree"
+	storagelog "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/log"
+	"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/writecache"
+	"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util"
+	apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status"
+	oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
+	"go.etcd.io/bbolt"
+	"go.uber.org/zap"
+)
+
+// store represents persistent storage with in-memory LRU cache
+// for flushed items on top of it.
+type store struct {
+	db *bbolt.DB
+}
+
+const dbName = "small.bolt"
+
+func (c *cache) openStore(readOnly bool) error {
+	err := util.MkdirAllX(c.path, os.ModePerm)
+	if err != nil {
+		return err
+	}
+
+	c.db, err = OpenDB(c.path, readOnly, c.openFile)
+	if err != nil {
+		return fmt.Errorf("could not open database: %w", err)
+	}
+
+	c.db.MaxBatchSize = c.maxBatchSize
+	c.db.MaxBatchDelay = c.maxBatchDelay
+
+	if !readOnly {
+		err = c.db.Update(func(tx *bbolt.Tx) error {
+			_, err := tx.CreateBucketIfNotExists(defaultBucket)
+			return err
+		})
+		if err != nil {
+			return fmt.Errorf("could not create default bucket: %w", err)
+		}
+	}
+
+	c.fsTree = fstree.New(
+		fstree.WithPath(c.path),
+		fstree.WithPerm(os.ModePerm),
+		fstree.WithDepth(1),
+		fstree.WithDirNameLen(1),
+		fstree.WithNoSync(c.noSync))
+	if err := c.fsTree.Open(readOnly); err != nil {
+		return fmt.Errorf("could not open FSTree: %w", err)
+	}
+
+	return nil
+}
+
+func (c *cache) deleteFromDB(keys []string) []string {
+	if len(keys) == 0 {
+		return keys
+	}
+
+	var errorIndex int
+	err := c.db.Batch(func(tx *bbolt.Tx) error {
+		b := tx.Bucket(defaultBucket)
+		for errorIndex = range keys {
+			if err := b.Delete([]byte(keys[errorIndex])); err != nil {
+				return err
+			}
+		}
+		return nil
+	})
+	for i := 0; i < errorIndex; i++ {
+		c.objCounters.DecDB()
+		c.metrics.Evict(writecache.StorageTypeDB)
+		storagelog.Write(c.log,
+			storagelog.AddressField(keys[i]),
+			storagelog.StorageTypeField(wcStorageType),
+			storagelog.OpField("db DELETE"),
+		)
+	}
+	if err != nil {
+		c.log.Error(logs.WritecacheCantRemoveObjectsFromTheDatabase, zap.Error(err))
+	}
+
+	copy(keys, keys[errorIndex:])
+	return keys[:len(keys)-errorIndex]
+}
+
+func (c *cache) deleteFromDisk(ctx context.Context, keys []string) []string {
+	if len(keys) == 0 {
+		return keys
+	}
+
+	var copyIndex int
+	var addr oid.Address
+
+	for i := range keys {
+		if err := addr.DecodeString(keys[i]); err != nil {
+			c.log.Error(logs.WritecacheCantParseAddress, zap.String("address", keys[i]))
+			continue
+		}
+
+		_, err := c.fsTree.Delete(ctx, common.DeletePrm{Address: addr})
+		if err != nil && !errors.As(err, new(apistatus.ObjectNotFound)) {
+			c.log.Error(logs.WritecacheCantRemoveObjectFromWritecache, zap.Error(err))
+
+			// Save the key for the next iteration.
+			keys[copyIndex] = keys[i]
+			copyIndex++
+			continue
+		} else if err == nil {
+			storagelog.Write(c.log,
+				storagelog.AddressField(keys[i]),
+				storagelog.StorageTypeField(wcStorageType),
+				storagelog.OpField("fstree DELETE"),
+			)
+			c.metrics.Evict(writecache.StorageTypeFSTree)
+			c.objCounters.DecFS()
+		}
+	}
+
+	return keys[:copyIndex]
+}

pkg/local_object_storage/writecache/writecachebbolt/util.go

@@ -0,0 +1,20 @@
+package writecachebbolt
+
+import (
+	"io/fs"
+	"os"
+	"path/filepath"
+	"time"
+
+	"go.etcd.io/bbolt"
+)
+
+// OpenDB opens BoltDB instance for write-cache. Opens in read-only mode if ro is true.
+func OpenDB(p string, ro bool, openFile func(string, int, fs.FileMode) (*os.File, error)) (*bbolt.DB, error) {
+	return bbolt.Open(filepath.Join(p, dbName), os.ModePerm, &bbolt.Options{
+		NoFreelistSync: true,
+		ReadOnly:       ro,
+		Timeout:        100 * time.Millisecond,
+		OpenFile:       openFile,
+	})
+}