[#569] writecache: Allow to seal writecache after flush

Signed-off-by: Dmitrii Stepanov <d.stepanov@yadro.com>

pkg/local_object_storage/writecache/flush.go

@@ -12,6 +12,7 @@ import (
 	"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common"
 	"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/metaerr"
 	meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase"
+	"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode"
 	"git.frostfs.info/TrueCloudLab/frostfs-observability/tracing"
 	objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
 	oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
@@ -33,6 +34,8 @@ const (
 	defaultFlushInterval = time.Second
 )
 
+var errIterationCompleted = errors.New("iteration completed")
+
 // runFlushLoop starts background workers which periodically flush objects to the blobstor.
 func (c *cache) runFlushLoop(ctx context.Context) {
 	if c.disableBackgroundFlush {
@@ -229,7 +232,7 @@ func (c *cache) workerFlushSmall(ctx context.Context) {
 			continue
 		}
 
-		c.deleteFromDB(objInfo.addr)
+		c.deleteFromDB(objInfo.addr, true)
 	}
 }
 
@@ -270,19 +273,29 @@ func (c *cache) flushObject(ctx context.Context, obj *objectSDK.Object, data []b
 }
 
 // Flush flushes all objects from the write-cache to the main storage.
-// Write-cache must be in readonly mode to ensure correctness of an operation and
-// to prevent interference with background flush workers.
-func (c *cache) Flush(ctx context.Context, ignoreErrors bool) error {
-	ctx, span := tracing.StartSpanFromContext(ctx, "Flush",
+func (c *cache) Flush(ctx context.Context, ignoreErrors, seal bool) error {
+	ctx, span := tracing.StartSpanFromContext(ctx, "writecache.Flush",
 		trace.WithAttributes(
 			attribute.Bool("ignore_errors", ignoreErrors),
+			attribute.Bool("seal", seal),
 		))
 	defer span.End()
 
-	c.modeMtx.RLock()
-	defer c.modeMtx.RUnlock()
+	c.modeMtx.Lock() // exclusive lock to not to conflict with background flush
+	defer c.modeMtx.Unlock()
 
-	return c.flush(ctx, ignoreErrors)
+	if err := c.flush(ctx, ignoreErrors); err != nil {
+		return err
+	}
+
+	if seal {
+		m := c.mode | mode.ReadOnly
+		if err := c.setMode(ctx, m, ignoreErrors); err != nil {
+			return err
+		}
+		c.metrics.SetMode(m)
+	}
+	return nil
 }
 
 func (c *cache) flush(ctx context.Context, ignoreErrors bool) error {
@@ -290,13 +303,53 @@ func (c *cache) flush(ctx context.Context, ignoreErrors bool) error {
 		return err
 	}
 
-	return c.db.View(func(tx *bbolt.Tx) error {
+	var last string
+	for {
+		batch, err := c.readNextDBBatch(ignoreErrors, last)
+		if err != nil {
+			return err
+		}
+		if len(batch) == 0 {
+			break
+		}
+		for _, item := range batch {
+			var obj objectSDK.Object
+			if err := obj.Unmarshal(item.data); err != nil {
+				c.reportFlushError(logs.FSTreeCantDecodeDBObjectAddress, item.address, metaerr.Wrap(err))
+				if ignoreErrors {
+					continue
+				}
+				return err
+			}
+
+			if err := c.flushObject(ctx, &obj, item.data, StorageTypeDB); err != nil {
+				return err
+			}
+			c.deleteFromDB(item.address, false)
+		}
+		last = batch[len(batch)-1].address
+	}
+	return nil
+}
+
+type batchItem struct {
+	data    []byte
+	address string
+}
+
+func (c *cache) readNextDBBatch(ignoreErrors bool, last string) ([]batchItem, error) {
+	const batchSize = 100
+	var batch []batchItem
+	err := c.db.View(func(tx *bbolt.Tx) error {
 		var addr oid.Address
 
 		b := tx.Bucket(defaultBucket)
 		cs := b.Cursor()
-		for k, data := cs.Seek(nil); k != nil; k, data = cs.Next() {
+		for k, data := cs.Seek([]byte(last)); k != nil; k, data = cs.Next() {
 			sa := string(k)
+			if sa == last {
+				continue
+			}
 			if err := addr.DecodeString(sa); err != nil {
 				c.reportFlushError(logs.FSTreeCantDecodeDBObjectAddress, sa, metaerr.Wrap(err))
 				if ignoreErrors {
@@ -305,19 +358,15 @@ func (c *cache) flush(ctx context.Context, ignoreErrors bool) error {
 				return err
 			}
 
-			var obj objectSDK.Object
-			if err := obj.Unmarshal(data); err != nil {
-				c.reportFlushError(logs.FSTreeCantDecodeDBObjectAddress, sa, metaerr.Wrap(err))
-				if ignoreErrors {
-					continue
-				}
-				return err
-			}
-
-			if err := c.flushObject(ctx, &obj, data, StorageTypeDB); err != nil {
-				return err
+			batch = append(batch, batchItem{data: bytes.Clone(data), address: sa})
+			if len(batch) == batchSize {
+				return errIterationCompleted
 			}
 		}
 		return nil
 	})
+	if err == nil || errors.Is(err, errIterationCompleted) {
+		return batch, nil
+	}
+	return nil, err
 }