[#569] cli: Add control shards writecache seal command

It does the same as `control shards flush-writecache --seal`, but
has better name.

Signed-off-by: Dmitrii Stepanov <d.stepanov@yadro.com>

cmd/frostfs-cli/modules/control/flush_cache.go

@@ -11,10 +11,11 @@ import (
 const sealFlag = "seal"
 
 var flushCacheCmd = &cobra.Command{
-	Use:   "flush-cache",
-	Short: "Flush objects from the write-cache to the main storage",
-	Long:  "Flush objects from the write-cache to the main storage",
-	Run:   flushCache,
+	Use:        "flush-cache",
+	Short:      "Flush objects from the write-cache to the main storage",
+	Long:       "Flush objects from the write-cache to the main storage",
+	Run:        flushCache,
+	Deprecated: "Flushing objects from writecache to the main storage is performed by writecache automatically. To flush and seal writecache use `frostfs-cli control shards writecache seal`.",
 }
 
 func flushCache(cmd *cobra.Command, _ []string) {

cmd/frostfs-cli/modules/control/shards.go

@@ -17,6 +17,7 @@ func initControlShardsCmd() {
 	shardsCmd.AddCommand(evacuationShardCmd)
 	shardsCmd.AddCommand(flushCacheCmd)
 	shardsCmd.AddCommand(doctorCmd)
+	shardsCmd.AddCommand(writecacheShardCmd)
 
 	initControlShardsListCmd()
 	initControlSetShardModeCmd()
@@ -24,4 +25,5 @@ func initControlShardsCmd() {
 	initControlEvacuationShardCmd()
 	initControlFlushCacheCmd()
 	initControlDoctorCmd()
+	initControlShardsWritecacheCmd()
 }

cmd/frostfs-cli/modules/control/writecache.go

@@ -0,0 +1,73 @@
+package control
+
+import (
+	"git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/rpc/client"
+	"git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/key"
+	commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common"
+	"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control"
+	"github.com/mr-tron/base58"
+	"github.com/spf13/cobra"
+)
+
+var writecacheShardCmd = &cobra.Command{
+	Use:   "writecache",
+	Short: "Operations with storage node's write-cache",
+	Long:  "Operations with storage node's write-cache",
+}
+
+var sealWritecacheShardCmd = &cobra.Command{
+	Use:   "seal",
+	Short: "Flush objects from write-cache and move write-cache to degraded read only mode.",
+	Long:  "Flush all the objects from the write-cache to the main storage and move the write-cache to the degraded read only mode: write-cache will be empty and no objects will be put in it.",
+	Run:   sealWritecache,
+}
+
+func sealWritecache(cmd *cobra.Command, _ []string) {
+	pk := key.Get(cmd)
+
+	ignoreErrors, _ := cmd.Flags().GetBool(ignoreErrorsFlag)
+
+	req := &control.SealWriteCacheRequest{Body: &control.SealWriteCacheRequest_Body{
+		Shard_ID:     getShardIDList(cmd),
+		IgnoreErrors: ignoreErrors,
+	}}
+
+	signRequest(cmd, pk, req)
+
+	cli := getClient(cmd, pk)
+
+	var resp *control.SealWriteCacheResponse
+	var err error
+	err = cli.ExecRaw(func(client *client.Client) error {
+		resp, err = control.SealWriteCache(client, req)
+		return err
+	})
+	commonCmd.ExitOnErr(cmd, "rpc error: %w", err)
+
+	verifyResponse(cmd, resp.GetSignature(), resp.GetBody())
+
+	var success, failed uint
+	for _, res := range resp.GetBody().GetResults() {
+		if res.GetSuccess() {
+			success++
+			cmd.Printf("Shard %s: OK\n", base58.Encode(res.GetShard_ID()))
+		} else {
+			failed++
+			cmd.Printf("Shard %s: failed with error %q\n", base58.Encode(res.GetShard_ID()), res.GetError())
+		}
+	}
+	cmd.Printf("Total: %d success, %d failed\n", success, failed)
+}
+
+func initControlShardsWritecacheCmd() {
+	writecacheShardCmd.AddCommand(sealWritecacheShardCmd)
+
+	initControlFlags(sealWritecacheShardCmd)
+
+	ff := sealWritecacheShardCmd.Flags()
+	ff.StringSlice(shardIDFlag, nil, "List of shard IDs in base58 encoding")
+	ff.Bool(shardAllFlag, false, "Process all shards")
+	ff.Bool(ignoreErrorsFlag, true, "Skip invalid/unreadable objects")
+
+	sealWritecacheShardCmd.MarkFlagsMutuallyExclusive(shardIDFlag, shardAllFlag)
+}

pkg/local_object_storage/engine/writecache.go

@@ -2,6 +2,7 @@ package engine
 
 import (
 	"context"
+	"sync"
 	"time"
 
 	"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard"
@@ -11,6 +12,7 @@ import (
 	"git.frostfs.info/TrueCloudLab/frostfs-observability/tracing"
 	"go.opentelemetry.io/otel/attribute"
 	"go.opentelemetry.io/otel/trace"
+	"golang.org/x/sync/errgroup"
 )
 
 // FlushWriteCachePrm groups the parameters of FlushWriteCache operation.
@@ -44,7 +46,7 @@ type FlushWriteCacheRes struct{}
 func (e *StorageEngine) FlushWriteCache(ctx context.Context, p FlushWriteCachePrm) (FlushWriteCacheRes, error) {
 	ctx, span := tracing.StartSpanFromContext(ctx, "StorageEngine.FlushWriteCache",
 		trace.WithAttributes(
-			attribute.String("shard)id", p.shardID.String()),
+			attribute.String("shard_id", p.shardID.String()),
 			attribute.Bool("ignore_errors", p.ignoreErrors),
 			attribute.Bool("seal", p.seal),
 		))
@@ -65,6 +67,79 @@ func (e *StorageEngine) FlushWriteCache(ctx context.Context, p FlushWriteCachePr
 	return FlushWriteCacheRes{}, sh.FlushWriteCache(ctx, prm)
 }
 
+type SealWriteCachePrm struct {
+	ShardIDs     []*shard.ID
+	IgnoreErrors bool
+}
+
+type ShardSealResult struct {
+	ShardID  *shard.ID
+	Success  bool
+	ErrorMsg string
+}
+
+type SealWriteCacheRes struct {
+	ShardResults []ShardSealResult
+}
+
+// SealWriteCache flushed all data to blobstore and moves write-cache to degraded read only mode.
+func (e *StorageEngine) SealWriteCache(ctx context.Context, prm SealWriteCachePrm) (SealWriteCacheRes, error) {
+	ctx, span := tracing.StartSpanFromContext(ctx, "StorageEngine.SealWriteCache",
+		trace.WithAttributes(
+			attribute.Int("shard_id_count", len(prm.ShardIDs)),
+			attribute.Bool("ignore_errors", prm.IgnoreErrors),
+		))
+	defer span.End()
+
+	res := SealWriteCacheRes{
+		ShardResults: make([]ShardSealResult, 0, len(prm.ShardIDs)),
+	}
+	resGuard := &sync.Mutex{}
+
+	eg, egCtx := errgroup.WithContext(ctx)
+	for _, shardID := range prm.ShardIDs {
+		shardID := shardID
+		eg.Go(func() error {
+			e.mtx.RLock()
+			sh, ok := e.shards[shardID.String()]
+			e.mtx.RUnlock()
+
+			if !ok {
+				resGuard.Lock()
+				defer resGuard.Unlock()
+				res.ShardResults = append(res.ShardResults, ShardSealResult{
+					ShardID:  shardID,
+					ErrorMsg: errShardNotFound.Error(),
+				})
+				return nil
+			}
+
+			err := sh.SealWriteCache(egCtx, shard.SealWriteCachePrm{IgnoreErrors: prm.IgnoreErrors})
+
+			resGuard.Lock()
+			defer resGuard.Unlock()
+
+			if err != nil {
+				res.ShardResults = append(res.ShardResults, ShardSealResult{
+					ShardID:  shardID,
+					ErrorMsg: err.Error(),
+				})
+			} else {
+				res.ShardResults = append(res.ShardResults, ShardSealResult{
+					ShardID: shardID,
+					Success: true,
+				})
+			}
+			return nil
+		})
+	}
+
+	if err := eg.Wait(); err != nil {
+		return SealWriteCacheRes{}, err
+	}
+	return res, nil
+}
+
 type writeCacheMetrics struct {
 	shardID string
 	metrics metrics.WriteCacheMetrics

pkg/local_object_storage/shard/writecache.go

@@ -56,3 +56,33 @@ func (s *Shard) FlushWriteCache(ctx context.Context, p FlushWriteCachePrm) error
 
 	return s.writeCache.Flush(ctx, p.ignoreErrors, p.seal)
 }
+
+type SealWriteCachePrm struct {
+	IgnoreErrors bool
+}
+
+// SealWriteCache flushes all data from the write-cache and moves it to degraded read only mode.
+func (s *Shard) SealWriteCache(ctx context.Context, p SealWriteCachePrm) error {
+	ctx, span := tracing.StartSpanFromContext(ctx, "Shard.SealWriteCache",
+		trace.WithAttributes(
+			attribute.String("shard_id", s.ID().String()),
+			attribute.Bool("ignore_errors", p.IgnoreErrors),
+		))
+	defer span.End()
+
+	if !s.hasWriteCache() {
+		return errWriteCacheDisabled
+	}
+
+	s.m.RLock()
+	defer s.m.RUnlock()
+
+	if s.info.Mode.ReadOnly() {
+		return ErrReadOnlyMode
+	}
+	if s.info.Mode.NoMetabase() {
+		return ErrDegradedMode
+	}
+
+	return s.writeCache.Seal(ctx, p.IgnoreErrors)
+}

pkg/local_object_storage/writecache/seal.go

@@ -0,0 +1,28 @@
+package writecache
+
+import (
+	"context"
+
+	"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode"
+	"git.frostfs.info/TrueCloudLab/frostfs-observability/tracing"
+	"go.opentelemetry.io/otel/attribute"
+	"go.opentelemetry.io/otel/trace"
+)
+
+func (c *cache) Seal(ctx context.Context, ignoreErrors bool) error {
+	ctx, span := tracing.StartSpanFromContext(ctx, "writecache.Seal",
+		trace.WithAttributes(
+			attribute.Bool("ignore_errors", ignoreErrors),
+		))
+	defer span.End()
+
+	c.modeMtx.Lock()
+	defer c.modeMtx.Unlock()
+
+	// flush will be done by setMode
+	err := c.setMode(ctx, mode.DegradedReadOnly, ignoreErrors)
+	if err == nil {
+		c.metrics.SetMode(mode.DegradedReadOnly)
+	}
+	return err
+}

pkg/local_object_storage/writecache/writecache.go

@@ -36,6 +36,7 @@ type Cache interface {
 	SetLogger(*logger.Logger)
 	DumpInfo() Info
 	Flush(context.Context, bool, bool) error
+	Seal(context.Context, bool) error
 
 	Init() error
 	Open(ctx context.Context, readOnly bool) error

pkg/services/control/rpc.go

@@ -24,6 +24,7 @@ const (
 	rpcGetChainLocalOverride    = "GetChainLocalOverride"
 	rpcListChainLocalOverrides  = "ListChainLocalOverrides"
 	rpcRemoveChainLocalOverride = "RemoveChainLocalOverride"
+	rpcSealWriteCache           = "SealWriteCache"
 )
 
 // HealthCheck executes ControlService.HealthCheck RPC.
@@ -264,3 +265,16 @@ func RemoveChainLocalOverride(cli *client.Client, req *RemoveChainLocalOverrideR
 
 	return wResp.message, nil
 }
+
+// SealWriteCache executes ControlService.SealWriteCache RPC.
+func SealWriteCache(cli *client.Client, req *SealWriteCacheRequest, opts ...client.CallOption) (*SealWriteCacheResponse, error) {
+	wResp := newResponseWrapper[SealWriteCacheResponse]()
+	wReq := &requestWrapper{m: req}
+
+	err := client.SendUnary(cli, common.CallMethodInfoUnary(serviceName, rpcSealWriteCache), wReq, wResp, opts...)
+	if err != nil {
+		return nil, err
+	}
+
+	return wResp.message, nil
+}

pkg/services/control/server/seal_writecache.go

@@ -0,0 +1,48 @@
+package control
+
+import (
+	"context"
+
+	"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/engine"
+	"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/status"
+)
+
+func (s *Server) SealWriteCache(ctx context.Context, req *control.SealWriteCacheRequest) (*control.SealWriteCacheResponse, error) {
+	err := s.isValidRequest(req)
+	if err != nil {
+		return nil, status.Error(codes.PermissionDenied, err.Error())
+	}
+
+	prm := engine.SealWriteCachePrm{
+		ShardIDs:     s.getShardIDList(req.GetBody().GetShard_ID()),
+		IgnoreErrors: req.GetBody().GetIgnoreErrors(),
+	}
+
+	res, err := s.s.SealWriteCache(ctx, prm)
+	if err != nil {
+		return nil, status.Error(codes.Internal, err.Error())
+	}
+
+	resp := &control.SealWriteCacheResponse{Body: &control.SealWriteCacheResponse_Body{}}
+	for _, r := range res.ShardResults {
+		if r.Success {
+			resp.Body.Results = append(resp.GetBody().GetResults(), &control.SealWriteCacheResponse_Body_Status{
+				Shard_ID: *r.ShardID,
+				Success:  true,
+			})
+		} else {
+			resp.Body.Results = append(resp.GetBody().GetResults(), &control.SealWriteCacheResponse_Body_Status{
+				Shard_ID: *r.ShardID,
+				Error:    r.ErrorMsg,
+			})
+		}
+	}
+
+	err = SignMessage(s.key, resp)
+	if err != nil {
+		return nil, err
+	}
+	return resp, nil
+}

pkg/services/control/service.proto

@@ -56,6 +56,9 @@ service ControlService {
 
     // Remove local access policy engine overrides stored in the node by chaind id.
     rpc RemoveChainLocalOverride (RemoveChainLocalOverrideRequest) returns (RemoveChainLocalOverrideResponse);
+
+    // Flush objects from write-cache and move it to degraded read only mode.
+    rpc SealWriteCache(SealWriteCacheRequest) returns (SealWriteCacheResponse);
 }
 
 // Health check request.
@@ -525,3 +528,32 @@ message RemoveChainLocalOverrideResponse {
 
     Signature signature = 2;
 }
+
+message SealWriteCacheRequest {
+    // Request body structure.
+    message Body {
+        // ID of the shard.
+        repeated bytes shard_ID = 1;
+
+        // Flag indicating whether object read errors should be ignored.
+        bool ignore_errors = 2;
+    }
+
+    Body body = 1;
+    Signature signature = 2;
+}
+
+message SealWriteCacheResponse {
+    message Body {
+        message Status {
+            bytes shard_ID = 1;
+            bool success = 2;
+            string error = 3;
+        }
+        repeated Status results = 1;
+    }
+
+    Body body = 1;
+
+    Signature signature = 2;
+}

pkg/services/control/service_frostfs.pb.go

@@ -2755,3 +2755,216 @@ func (x *RemoveChainLocalOverrideResponse) ReadSignedData(buf []byte) ([]byte, e
 func (x *RemoveChainLocalOverrideResponse) SetSignature(sig *Signature) {
 	x.Signature = sig
 }
+
+// StableSize returns the size of x in protobuf format.
+//
+// Structures with the same field values have the same binary size.
+func (x *SealWriteCacheRequest_Body) StableSize() (size int) {
+	if x == nil {
+		return 0
+	}
+	size += proto.RepeatedBytesSize(1, x.Shard_ID)
+	size += proto.BoolSize(2, x.IgnoreErrors)
+	return size
+}
+
+// StableMarshal marshals x in protobuf binary format with stable field order.
+//
+// If buffer length is less than x.StableSize(), new buffer is allocated.
+//
+// Returns any error encountered which did not allow writing the data completely.
+// Otherwise, returns the buffer in which the data is written.
+//
+// Structures with the same field values have the same binary format.
+func (x *SealWriteCacheRequest_Body) StableMarshal(buf []byte) []byte {
+	if x == nil {
+		return []byte{}
+	}
+	if buf == nil {
+		buf = make([]byte, x.StableSize())
+	}
+	var offset int
+	offset += proto.RepeatedBytesMarshal(1, buf[offset:], x.Shard_ID)
+	offset += proto.BoolMarshal(2, buf[offset:], x.IgnoreErrors)
+	return buf
+}
+
+// StableSize returns the size of x in protobuf format.
+//
+// Structures with the same field values have the same binary size.
+func (x *SealWriteCacheRequest) StableSize() (size int) {
+	if x == nil {
+		return 0
+	}
+	size += proto.NestedStructureSize(1, x.Body)
+	size += proto.NestedStructureSize(2, x.Signature)
+	return size
+}
+
+// StableMarshal marshals x in protobuf binary format with stable field order.
+//
+// If buffer length is less than x.StableSize(), new buffer is allocated.
+//
+// Returns any error encountered which did not allow writing the data completely.
+// Otherwise, returns the buffer in which the data is written.
+//
+// Structures with the same field values have the same binary format.
+func (x *SealWriteCacheRequest) StableMarshal(buf []byte) []byte {
+	if x == nil {
+		return []byte{}
+	}
+	if buf == nil {
+		buf = make([]byte, x.StableSize())
+	}
+	var offset int
+	offset += proto.NestedStructureMarshal(1, buf[offset:], x.Body)
+	offset += proto.NestedStructureMarshal(2, buf[offset:], x.Signature)
+	return buf
+}
+
+// ReadSignedData fills buf with signed data of x.
+// If buffer length is less than x.SignedDataSize(), new buffer is allocated.
+//
+// Returns any error encountered which did not allow writing the data completely.
+// Otherwise, returns the buffer in which the data is written.
+//
+// Structures with the same field values have the same signed data.
+func (x *SealWriteCacheRequest) SignedDataSize() int {
+	return x.GetBody().StableSize()
+}
+
+// SignedDataSize returns size of the request signed data in bytes.
+//
+// Structures with the same field values have the same signed data size.
+func (x *SealWriteCacheRequest) ReadSignedData(buf []byte) ([]byte, error) {
+	return x.GetBody().StableMarshal(buf), nil
+}
+
+func (x *SealWriteCacheRequest) SetSignature(sig *Signature) {
+	x.Signature = sig
+}
+
+// StableSize returns the size of x in protobuf format.
+//
+// Structures with the same field values have the same binary size.
+func (x *SealWriteCacheResponse_Body_Status) StableSize() (size int) {
+	if x == nil {
+		return 0
+	}
+	size += proto.BytesSize(1, x.Shard_ID)
+	size += proto.BoolSize(2, x.Success)
+	size += proto.StringSize(3, x.Error)
+	return size
+}
+
+// StableMarshal marshals x in protobuf binary format with stable field order.
+//
+// If buffer length is less than x.StableSize(), new buffer is allocated.
+//
+// Returns any error encountered which did not allow writing the data completely.
+// Otherwise, returns the buffer in which the data is written.
+//
+// Structures with the same field values have the same binary format.
+func (x *SealWriteCacheResponse_Body_Status) StableMarshal(buf []byte) []byte {
+	if x == nil {
+		return []byte{}
+	}
+	if buf == nil {
+		buf = make([]byte, x.StableSize())
+	}
+	var offset int
+	offset += proto.BytesMarshal(1, buf[offset:], x.Shard_ID)
+	offset += proto.BoolMarshal(2, buf[offset:], x.Success)
+	offset += proto.StringMarshal(3, buf[offset:], x.Error)
+	return buf
+}
+
+// StableSize returns the size of x in protobuf format.
+//
+// Structures with the same field values have the same binary size.
+func (x *SealWriteCacheResponse_Body) StableSize() (size int) {
+	if x == nil {
+		return 0
+	}
+	for i := range x.Results {
+		size += proto.NestedStructureSize(1, x.Results[i])
+	}
+	return size
+}
+
+// StableMarshal marshals x in protobuf binary format with stable field order.
+//
+// If buffer length is less than x.StableSize(), new buffer is allocated.
+//
+// Returns any error encountered which did not allow writing the data completely.
+// Otherwise, returns the buffer in which the data is written.
+//
+// Structures with the same field values have the same binary format.
+func (x *SealWriteCacheResponse_Body) StableMarshal(buf []byte) []byte {
+	if x == nil {
+		return []byte{}
+	}
+	if buf == nil {
+		buf = make([]byte, x.StableSize())
+	}
+	var offset int
+	for i := range x.Results {
+		offset += proto.NestedStructureMarshal(1, buf[offset:], x.Results[i])
+	}
+	return buf
+}
+
+// StableSize returns the size of x in protobuf format.
+//
+// Structures with the same field values have the same binary size.
+func (x *SealWriteCacheResponse) StableSize() (size int) {
+	if x == nil {
+		return 0
+	}
+	size += proto.NestedStructureSize(1, x.Body)
+	size += proto.NestedStructureSize(2, x.Signature)
+	return size
+}
+
+// StableMarshal marshals x in protobuf binary format with stable field order.
+//
+// If buffer length is less than x.StableSize(), new buffer is allocated.
+//
+// Returns any error encountered which did not allow writing the data completely.
+// Otherwise, returns the buffer in which the data is written.
+//
+// Structures with the same field values have the same binary format.
+func (x *SealWriteCacheResponse) StableMarshal(buf []byte) []byte {
+	if x == nil {
+		return []byte{}
+	}
+	if buf == nil {
+		buf = make([]byte, x.StableSize())
+	}
+	var offset int
+	offset += proto.NestedStructureMarshal(1, buf[offset:], x.Body)
+	offset += proto.NestedStructureMarshal(2, buf[offset:], x.Signature)
+	return buf
+}
+
+// ReadSignedData fills buf with signed data of x.
+// If buffer length is less than x.SignedDataSize(), new buffer is allocated.
+//
+// Returns any error encountered which did not allow writing the data completely.
+// Otherwise, returns the buffer in which the data is written.
+//
+// Structures with the same field values have the same signed data.
+func (x *SealWriteCacheResponse) SignedDataSize() int {
+	return x.GetBody().StableSize()
+}
+
+// SignedDataSize returns size of the request signed data in bytes.
+//
+// Structures with the same field values have the same signed data size.
+func (x *SealWriteCacheResponse) ReadSignedData(buf []byte) ([]byte, error) {
+	return x.GetBody().StableMarshal(buf), nil
+}
+
+func (x *SealWriteCacheResponse) SetSignature(sig *Signature) {
+	x.Signature = sig
+}

pkg/services/control/service_grpc.pb.go

@@ -35,6 +35,7 @@ const (
 	ControlService_GetChainLocalOverride_FullMethodName    = "/control.ControlService/GetChainLocalOverride"
 	ControlService_ListChainLocalOverrides_FullMethodName  = "/control.ControlService/ListChainLocalOverrides"
 	ControlService_RemoveChainLocalOverride_FullMethodName = "/control.ControlService/RemoveChainLocalOverride"
+	ControlService_SealWriteCache_FullMethodName           = "/control.ControlService/SealWriteCache"
 )
 
 // ControlServiceClient is the client API for ControlService service.
@@ -74,6 +75,8 @@ type ControlServiceClient interface {
 	ListChainLocalOverrides(ctx context.Context, in *ListChainLocalOverridesRequest, opts ...grpc.CallOption) (*ListChainLocalOverridesResponse, error)
 	// Remove local access policy engine overrides stored in the node by chaind id.
 	RemoveChainLocalOverride(ctx context.Context, in *RemoveChainLocalOverrideRequest, opts ...grpc.CallOption) (*RemoveChainLocalOverrideResponse, error)
+	// Flush objects from write-cache and move it to degraded read only mode.
+	SealWriteCache(ctx context.Context, in *SealWriteCacheRequest, opts ...grpc.CallOption) (*SealWriteCacheResponse, error)
 }
 
 type controlServiceClient struct {
@@ -228,6 +231,15 @@ func (c *controlServiceClient) RemoveChainLocalOverride(ctx context.Context, in
 	return out, nil
 }
 
+func (c *controlServiceClient) SealWriteCache(ctx context.Context, in *SealWriteCacheRequest, opts ...grpc.CallOption) (*SealWriteCacheResponse, error) {
+	out := new(SealWriteCacheResponse)
+	err := c.cc.Invoke(ctx, ControlService_SealWriteCache_FullMethodName, in, out, opts...)
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
 // ControlServiceServer is the server API for ControlService service.
 // All implementations should embed UnimplementedControlServiceServer
 // for forward compatibility
@@ -265,6 +277,8 @@ type ControlServiceServer interface {
 	ListChainLocalOverrides(context.Context, *ListChainLocalOverridesRequest) (*ListChainLocalOverridesResponse, error)
 	// Remove local access policy engine overrides stored in the node by chaind id.
 	RemoveChainLocalOverride(context.Context, *RemoveChainLocalOverrideRequest) (*RemoveChainLocalOverrideResponse, error)
+	// Flush objects from write-cache and move it to degraded read only mode.
+	SealWriteCache(context.Context, *SealWriteCacheRequest) (*SealWriteCacheResponse, error)
 }
 
 // UnimplementedControlServiceServer should be embedded to have forward compatible implementations.
@@ -319,6 +333,9 @@ func (UnimplementedControlServiceServer) ListChainLocalOverrides(context.Context
 func (UnimplementedControlServiceServer) RemoveChainLocalOverride(context.Context, *RemoveChainLocalOverrideRequest) (*RemoveChainLocalOverrideResponse, error) {
 	return nil, status.Errorf(codes.Unimplemented, "method RemoveChainLocalOverride not implemented")
 }
+func (UnimplementedControlServiceServer) SealWriteCache(context.Context, *SealWriteCacheRequest) (*SealWriteCacheResponse, error) {
+	return nil, status.Errorf(codes.Unimplemented, "method SealWriteCache not implemented")
+}
 
 // UnsafeControlServiceServer may be embedded to opt out of forward compatibility for this service.
 // Use of this interface is not recommended, as added methods to ControlServiceServer will
@@ -619,6 +636,24 @@ func _ControlService_RemoveChainLocalOverride_Handler(srv interface{}, ctx conte
 	return interceptor(ctx, in, info, handler)
 }
 
+func _ControlService_SealWriteCache_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+	in := new(SealWriteCacheRequest)
+	if err := dec(in); err != nil {
+		return nil, err
+	}
+	if interceptor == nil {
+		return srv.(ControlServiceServer).SealWriteCache(ctx, in)
+	}
+	info := &grpc.UnaryServerInfo{
+		Server:     srv,
+		FullMethod: ControlService_SealWriteCache_FullMethodName,
+	}
+	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
+		return srv.(ControlServiceServer).SealWriteCache(ctx, req.(*SealWriteCacheRequest))
+	}
+	return interceptor(ctx, in, info, handler)
+}
+
 // ControlService_ServiceDesc is the grpc.ServiceDesc for ControlService service.
 // It's only intended for direct use with grpc.RegisterService,
 // and not to be introspected or modified (even as a copy)
@@ -690,6 +725,10 @@ var ControlService_ServiceDesc = grpc.ServiceDesc{
 			MethodName: "RemoveChainLocalOverride",
 			Handler:    _ControlService_RemoveChainLocalOverride_Handler,
 		},
+		{
+			MethodName: "SealWriteCache",
+			Handler:    _ControlService_SealWriteCache_Handler,
+		},
 	},
 	Streams:  []grpc.StreamDesc{},
 	Metadata: "pkg/services/control/service.proto",