From 581887148a8971434a3becbc4e561c8a859f5fd7 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Wed, 27 Dec 2023 14:37:22 +0300 Subject: [PATCH] [#569] cli: Add `control shards writecache seal` command It does the same as `control shards flush-writecache --seal`, but has better name. Signed-off-by: Dmitrii Stepanov --- .../modules/control/flush_cache.go | 9 +- cmd/frostfs-cli/modules/control/shards.go | 2 + cmd/frostfs-cli/modules/control/writecache.go | 73 +++++++++++++++++ pkg/local_object_storage/engine/writecache.go | 77 +++++++++++++++++- pkg/local_object_storage/shard/writecache.go | 30 +++++++ pkg/local_object_storage/writecache/seal.go | 28 +++++++ .../writecache/writecache.go | 1 + pkg/services/control/rpc.go | 14 ++++ .../control/server/seal_writecache.go | 48 +++++++++++ pkg/services/control/service.pb.go | Bin 195243 -> 209551 bytes pkg/services/control/service.proto | 32 ++++++++ pkg/services/control/service_frostfs.pb.go | Bin 89904 -> 96691 bytes pkg/services/control/service_grpc.pb.go | Bin 31613 -> 33410 bytes 13 files changed, 309 insertions(+), 5 deletions(-) create mode 100644 cmd/frostfs-cli/modules/control/writecache.go create mode 100644 pkg/local_object_storage/writecache/seal.go create mode 100644 pkg/services/control/server/seal_writecache.go diff --git a/cmd/frostfs-cli/modules/control/flush_cache.go b/cmd/frostfs-cli/modules/control/flush_cache.go index 7f632e9fc..541961903 100644 --- a/cmd/frostfs-cli/modules/control/flush_cache.go +++ b/cmd/frostfs-cli/modules/control/flush_cache.go @@ -11,10 +11,11 @@ import ( const sealFlag = "seal" var flushCacheCmd = &cobra.Command{ - Use: "flush-cache", - Short: "Flush objects from the write-cache to the main storage", - Long: "Flush objects from the write-cache to the main storage", - Run: flushCache, + Use: "flush-cache", + Short: "Flush objects from the write-cache to the main storage", + Long: "Flush objects from the write-cache to the main storage", + Run: flushCache, + Deprecated: "Flushing objects from writecache to the main storage is performed by writecache automatically. To flush and seal writecache use `frostfs-cli control shards writecache seal`.", } func flushCache(cmd *cobra.Command, _ []string) { diff --git a/cmd/frostfs-cli/modules/control/shards.go b/cmd/frostfs-cli/modules/control/shards.go index 6208c560b..6d3ef420c 100644 --- a/cmd/frostfs-cli/modules/control/shards.go +++ b/cmd/frostfs-cli/modules/control/shards.go @@ -17,6 +17,7 @@ func initControlShardsCmd() { shardsCmd.AddCommand(evacuationShardCmd) shardsCmd.AddCommand(flushCacheCmd) shardsCmd.AddCommand(doctorCmd) + shardsCmd.AddCommand(writecacheShardCmd) initControlShardsListCmd() initControlSetShardModeCmd() @@ -24,4 +25,5 @@ func initControlShardsCmd() { initControlEvacuationShardCmd() initControlFlushCacheCmd() initControlDoctorCmd() + initControlShardsWritecacheCmd() } diff --git a/cmd/frostfs-cli/modules/control/writecache.go b/cmd/frostfs-cli/modules/control/writecache.go new file mode 100644 index 000000000..abc4ed2e6 --- /dev/null +++ b/cmd/frostfs-cli/modules/control/writecache.go @@ -0,0 +1,73 @@ +package control + +import ( + "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/rpc/client" + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/key" + commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control" + "github.com/mr-tron/base58" + "github.com/spf13/cobra" +) + +var writecacheShardCmd = &cobra.Command{ + Use: "writecache", + Short: "Operations with storage node's write-cache", + Long: "Operations with storage node's write-cache", +} + +var sealWritecacheShardCmd = &cobra.Command{ + Use: "seal", + Short: "Flush objects from write-cache and move write-cache to degraded read only mode.", + Long: "Flush all the objects from the write-cache to the main storage and move the write-cache to the degraded read only mode: write-cache will be empty and no objects will be put in it.", + Run: sealWritecache, +} + +func sealWritecache(cmd *cobra.Command, _ []string) { + pk := key.Get(cmd) + + ignoreErrors, _ := cmd.Flags().GetBool(ignoreErrorsFlag) + + req := &control.SealWriteCacheRequest{Body: &control.SealWriteCacheRequest_Body{ + Shard_ID: getShardIDList(cmd), + IgnoreErrors: ignoreErrors, + }} + + signRequest(cmd, pk, req) + + cli := getClient(cmd, pk) + + var resp *control.SealWriteCacheResponse + var err error + err = cli.ExecRaw(func(client *client.Client) error { + resp, err = control.SealWriteCache(client, req) + return err + }) + commonCmd.ExitOnErr(cmd, "rpc error: %w", err) + + verifyResponse(cmd, resp.GetSignature(), resp.GetBody()) + + var success, failed uint + for _, res := range resp.GetBody().GetResults() { + if res.GetSuccess() { + success++ + cmd.Printf("Shard %s: OK\n", base58.Encode(res.GetShard_ID())) + } else { + failed++ + cmd.Printf("Shard %s: failed with error %q\n", base58.Encode(res.GetShard_ID()), res.GetError()) + } + } + cmd.Printf("Total: %d success, %d failed\n", success, failed) +} + +func initControlShardsWritecacheCmd() { + writecacheShardCmd.AddCommand(sealWritecacheShardCmd) + + initControlFlags(sealWritecacheShardCmd) + + ff := sealWritecacheShardCmd.Flags() + ff.StringSlice(shardIDFlag, nil, "List of shard IDs in base58 encoding") + ff.Bool(shardAllFlag, false, "Process all shards") + ff.Bool(ignoreErrorsFlag, true, "Skip invalid/unreadable objects") + + sealWritecacheShardCmd.MarkFlagsMutuallyExclusive(shardIDFlag, shardAllFlag) +} diff --git a/pkg/local_object_storage/engine/writecache.go b/pkg/local_object_storage/engine/writecache.go index d92a86f5d..0eca018f8 100644 --- a/pkg/local_object_storage/engine/writecache.go +++ b/pkg/local_object_storage/engine/writecache.go @@ -2,6 +2,7 @@ package engine import ( "context" + "sync" "time" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard" @@ -11,6 +12,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" + "golang.org/x/sync/errgroup" ) // FlushWriteCachePrm groups the parameters of FlushWriteCache operation. @@ -44,7 +46,7 @@ type FlushWriteCacheRes struct{} func (e *StorageEngine) FlushWriteCache(ctx context.Context, p FlushWriteCachePrm) (FlushWriteCacheRes, error) { ctx, span := tracing.StartSpanFromContext(ctx, "StorageEngine.FlushWriteCache", trace.WithAttributes( - attribute.String("shard)id", p.shardID.String()), + attribute.String("shard_id", p.shardID.String()), attribute.Bool("ignore_errors", p.ignoreErrors), attribute.Bool("seal", p.seal), )) @@ -65,6 +67,79 @@ func (e *StorageEngine) FlushWriteCache(ctx context.Context, p FlushWriteCachePr return FlushWriteCacheRes{}, sh.FlushWriteCache(ctx, prm) } +type SealWriteCachePrm struct { + ShardIDs []*shard.ID + IgnoreErrors bool +} + +type ShardSealResult struct { + ShardID *shard.ID + Success bool + ErrorMsg string +} + +type SealWriteCacheRes struct { + ShardResults []ShardSealResult +} + +// SealWriteCache flushed all data to blobstore and moves write-cache to degraded read only mode. +func (e *StorageEngine) SealWriteCache(ctx context.Context, prm SealWriteCachePrm) (SealWriteCacheRes, error) { + ctx, span := tracing.StartSpanFromContext(ctx, "StorageEngine.SealWriteCache", + trace.WithAttributes( + attribute.Int("shard_id_count", len(prm.ShardIDs)), + attribute.Bool("ignore_errors", prm.IgnoreErrors), + )) + defer span.End() + + res := SealWriteCacheRes{ + ShardResults: make([]ShardSealResult, 0, len(prm.ShardIDs)), + } + resGuard := &sync.Mutex{} + + eg, egCtx := errgroup.WithContext(ctx) + for _, shardID := range prm.ShardIDs { + shardID := shardID + eg.Go(func() error { + e.mtx.RLock() + sh, ok := e.shards[shardID.String()] + e.mtx.RUnlock() + + if !ok { + resGuard.Lock() + defer resGuard.Unlock() + res.ShardResults = append(res.ShardResults, ShardSealResult{ + ShardID: shardID, + ErrorMsg: errShardNotFound.Error(), + }) + return nil + } + + err := sh.SealWriteCache(egCtx, shard.SealWriteCachePrm{IgnoreErrors: prm.IgnoreErrors}) + + resGuard.Lock() + defer resGuard.Unlock() + + if err != nil { + res.ShardResults = append(res.ShardResults, ShardSealResult{ + ShardID: shardID, + ErrorMsg: err.Error(), + }) + } else { + res.ShardResults = append(res.ShardResults, ShardSealResult{ + ShardID: shardID, + Success: true, + }) + } + return nil + }) + } + + if err := eg.Wait(); err != nil { + return SealWriteCacheRes{}, err + } + return res, nil +} + type writeCacheMetrics struct { shardID string metrics metrics.WriteCacheMetrics diff --git a/pkg/local_object_storage/shard/writecache.go b/pkg/local_object_storage/shard/writecache.go index 4e57a0497..05e014d29 100644 --- a/pkg/local_object_storage/shard/writecache.go +++ b/pkg/local_object_storage/shard/writecache.go @@ -56,3 +56,33 @@ func (s *Shard) FlushWriteCache(ctx context.Context, p FlushWriteCachePrm) error return s.writeCache.Flush(ctx, p.ignoreErrors, p.seal) } + +type SealWriteCachePrm struct { + IgnoreErrors bool +} + +// SealWriteCache flushes all data from the write-cache and moves it to degraded read only mode. +func (s *Shard) SealWriteCache(ctx context.Context, p SealWriteCachePrm) error { + ctx, span := tracing.StartSpanFromContext(ctx, "Shard.SealWriteCache", + trace.WithAttributes( + attribute.String("shard_id", s.ID().String()), + attribute.Bool("ignore_errors", p.IgnoreErrors), + )) + defer span.End() + + if !s.hasWriteCache() { + return errWriteCacheDisabled + } + + s.m.RLock() + defer s.m.RUnlock() + + if s.info.Mode.ReadOnly() { + return ErrReadOnlyMode + } + if s.info.Mode.NoMetabase() { + return ErrDegradedMode + } + + return s.writeCache.Seal(ctx, p.IgnoreErrors) +} diff --git a/pkg/local_object_storage/writecache/seal.go b/pkg/local_object_storage/writecache/seal.go new file mode 100644 index 000000000..be4fec367 --- /dev/null +++ b/pkg/local_object_storage/writecache/seal.go @@ -0,0 +1,28 @@ +package writecache + +import ( + "context" + + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" +) + +func (c *cache) Seal(ctx context.Context, ignoreErrors bool) error { + ctx, span := tracing.StartSpanFromContext(ctx, "writecache.Seal", + trace.WithAttributes( + attribute.Bool("ignore_errors", ignoreErrors), + )) + defer span.End() + + c.modeMtx.Lock() + defer c.modeMtx.Unlock() + + // flush will be done by setMode + err := c.setMode(ctx, mode.DegradedReadOnly, ignoreErrors) + if err == nil { + c.metrics.SetMode(mode.DegradedReadOnly) + } + return err +} diff --git a/pkg/local_object_storage/writecache/writecache.go b/pkg/local_object_storage/writecache/writecache.go index 0549c27f7..99fc5390a 100644 --- a/pkg/local_object_storage/writecache/writecache.go +++ b/pkg/local_object_storage/writecache/writecache.go @@ -36,6 +36,7 @@ type Cache interface { SetLogger(*logger.Logger) DumpInfo() Info Flush(context.Context, bool, bool) error + Seal(context.Context, bool) error Init() error Open(ctx context.Context, readOnly bool) error diff --git a/pkg/services/control/rpc.go b/pkg/services/control/rpc.go index b9ec05e71..3fd4d4ae9 100644 --- a/pkg/services/control/rpc.go +++ b/pkg/services/control/rpc.go @@ -24,6 +24,7 @@ const ( rpcGetChainLocalOverride = "GetChainLocalOverride" rpcListChainLocalOverrides = "ListChainLocalOverrides" rpcRemoveChainLocalOverride = "RemoveChainLocalOverride" + rpcSealWriteCache = "SealWriteCache" ) // HealthCheck executes ControlService.HealthCheck RPC. @@ -264,3 +265,16 @@ func RemoveChainLocalOverride(cli *client.Client, req *RemoveChainLocalOverrideR return wResp.message, nil } + +// SealWriteCache executes ControlService.SealWriteCache RPC. +func SealWriteCache(cli *client.Client, req *SealWriteCacheRequest, opts ...client.CallOption) (*SealWriteCacheResponse, error) { + wResp := newResponseWrapper[SealWriteCacheResponse]() + wReq := &requestWrapper{m: req} + + err := client.SendUnary(cli, common.CallMethodInfoUnary(serviceName, rpcSealWriteCache), wReq, wResp, opts...) + if err != nil { + return nil, err + } + + return wResp.message, nil +} diff --git a/pkg/services/control/server/seal_writecache.go b/pkg/services/control/server/seal_writecache.go new file mode 100644 index 000000000..5c39cf484 --- /dev/null +++ b/pkg/services/control/server/seal_writecache.go @@ -0,0 +1,48 @@ +package control + +import ( + "context" + + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/engine" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +func (s *Server) SealWriteCache(ctx context.Context, req *control.SealWriteCacheRequest) (*control.SealWriteCacheResponse, error) { + err := s.isValidRequest(req) + if err != nil { + return nil, status.Error(codes.PermissionDenied, err.Error()) + } + + prm := engine.SealWriteCachePrm{ + ShardIDs: s.getShardIDList(req.GetBody().GetShard_ID()), + IgnoreErrors: req.GetBody().GetIgnoreErrors(), + } + + res, err := s.s.SealWriteCache(ctx, prm) + if err != nil { + return nil, status.Error(codes.Internal, err.Error()) + } + + resp := &control.SealWriteCacheResponse{Body: &control.SealWriteCacheResponse_Body{}} + for _, r := range res.ShardResults { + if r.Success { + resp.Body.Results = append(resp.GetBody().GetResults(), &control.SealWriteCacheResponse_Body_Status{ + Shard_ID: *r.ShardID, + Success: true, + }) + } else { + resp.Body.Results = append(resp.GetBody().GetResults(), &control.SealWriteCacheResponse_Body_Status{ + Shard_ID: *r.ShardID, + Error: r.ErrorMsg, + }) + } + } + + err = SignMessage(s.key, resp) + if err != nil { + return nil, err + } + return resp, nil +} diff --git a/pkg/services/control/service.pb.go b/pkg/services/control/service.pb.go index 90e643e2957a7b3ee9edca179999666ef1a3047b..33ed898ba548d45a050294d5b310848da06ff1f9 100644 GIT binary patch delta 9779 zcmb7Kd0bTG-v2y57+{8NP()cBaYw`%mKnee#2ph*D{+N%NK0WvFf|pZwA|67{Pem~ znxeSADAT zKH=)8gwAiYJ$J^Vh0}}dX}MFT*_GR4R{1sqDF^lyE2e3Wv_$MMKg6q^RN42~E}N89c{!??P52uhU)O>lDf%i7TI+9T6Az06Vql1Vpy=ZW1oV@>Y5aj^%IV z9Gw11wosn=+OBME<*Q6MXHfcm`n_`Z>Q$vQP(2^)`S~kuiut--xpz5){RfIP<+zV8 zSMMbcrN*dJsxEH8t-CU2=@%ereK6(n01xGvTDx-jtMIjTX-fa#AjQ;Hr~LaDPp2T2^Dlt+3fwKqMb%z;Wuu(1J<;w{=M zE58lps(dk7skw6v^cs4GyYAqDn;(;s`K!C~ub*?36H&fON;jSIOYP5qT~&7V^i|4l z84ljM{R}C`@7jUTnzx-J5B8H%!lSj5QhLWj$-J{h>HWK;QGq(;z|HQ;U%UD$bv->) znuDgF-WB|$yU1H#D|{6H1fBBEPda7tFLiuLC$6nG0l4UAgVOW!?(1`e&%x6Fq!Z^F zRP=q|2u&rZUhOT&c0SrQ?# z+&HACitMjs=6G{Pr1*uG#)zvMoI2&W? zFX3aCL(}5LHO}vai`AVWr*D+FL>!VR>`t&U2ws>30k|)AdvaC+o%V)GKa^HfoDJx#HKXitz7=G zJ8$eKB$+SgIK))vKL?9tf_qIAB}&FJoszjkr`)R@#6FWmNCUuC!)@6ywHY*b*X>C{ zG@v2hdDX*>7|xV*@r-AMUAS}R6tRST2O?jjnc%@nyRcKK9cR1UUE zlTy0ymR#Y*Tc;vZvZsn&64gb&IalbNt@FfT;^${WQ0e**hKEP+#_8e+2NollZ*FO$ zWX{(~u#P9Ch#s8c5UuM$o@)Jg&JZPnX}-##&EN8cQSjQQRpL;m(IMuz@xW3Pm*d4S zUcEsKS0=yGmTT(Bn^QIl57`ylvftDAsE6h9*)ovl?+D{shtSCeq72-t-7EQ@+nxWM zBW!M*zC>l(wxUui*fcN>SE@eNaov1T$rqkS4&B?MLS(?fLc5*h5RW_47m8nr%U23_ z<+)W;mDLA=Ro&QFCjKtLSCwV2HW8omW)tx*JS*NKer*j5_IX3&Sz|ytr7NWSWFtZUbXSfX4Pk&(AE1f4D^>q! zqnO{;NucKa;j6_o6yH|RpM1YPZ(FPNmNYJ1D?a3d+Zq_PVe?v~K9bADC{#suZeJ&! zbW_$?>`uo9VG-&;s{5#Yqw7zc4N*g*x_;s&@iqyY65unHfgjT7{41wT_0Eq_!p~?2 z^V1?%Sd}mCTvh5mlE`Z9DJRZq*9b)(&#Dy9adi(Q*!*A-+I&Vk%Y@Jip4>;g+c5k& zwX5jZ9F1%J#Wcuyb$(tmwAO0ZY1~RAX=|3l;yy5~xsf%I15hHq^$VM8*Slt`UzAiP zGV{aYVxvvw2P`3)sQsQIyBQrHP8DNX70q+lO$~dQl)y21@j}HG;of$3@rVm3gRM7Ps}dJPmXj!m06{YODEU#Lv;`Ys;F{0HF^>ta}$T*7_8LX8%Nfp`?>1OkJjeMfoh8li;Mdc zZ%L^UL+nMPG@H70e}D(LV5|uDbYUr?ckBCY=Blrh4jmVGjh>cUICb){fm~JyolhIQ zxoD(F_8(n5Wr}_FY?JA+f`S<)laljQMBHo**20_h(1Mv*`)4|e=0)Zh^n(fu3kppp zwB;-j+$k=)z)=*PpF7ju*Dfzwa8O|~8qG6G({W8&pih$})viLha=92!HnR-NA8(bt z9^I6jtcd%uv!u7bA2>HU7wqm6D@|P1I?=twR<3DR>6h0@joO0)*O;5e?k2>q-r)$K za^|~^oYF&dx*zY>8n)(hAj?1~uSQ0AXb`Tc6m8o6@fD^~oIz!vMuKAkMVHp+kj$FL}s9~K42#Gq756r67TS6FKW$sRiYK=Ul4&(R%Q^j;mhBnoDbg+ zT{+|j=rQvR;mxmqF9P`91yI&}FZ?@dM(3(_m{prxHd{5D;Rep#Cj$SA*%l5vEV^^n zev!xv55vrt_ltp1(uG!w>YOPmyFYcEjaLk&FusyVdamwFR!(V09?nhv6eqlLHPtL^ z=}Iwi@oGRcMa@llCa6)f$qq8trXpEY!M1olVy5og`*SR#hENLR+IylGJB}knhjs~{ zKeNtOBg|a$x`^hQOTx!P+GI=Oqjy9wpWcfBzCM`(*#D65G&K|0L=%geMTqpv-qeE~ zFJlL**$c^?zY?J`R^of;a{fbMR z26EL65zDp71^EfKL(rIrBfuQ92FL>%Y@CBsHgQ zr@&qCY&(8?AUZ+BCnAMg4XS5ceM-b}=3w%X{KRQ7kbi#!ZX2`{{Q{K{!_nu^Ada3y zAcwoKa?gp>CTbrTOs>S4%Hl`SYkfI<*l#FBu>TN5Va*uo&DodmTCwMI1hzJlJn;s3 zvM-=xFNvOfYcR|!zbLHUS{9?0p~;q^l%SbfD?)kE&mywI8)oj@FOsC979A`xCP2`( zA4Hpa#16b818G*;hdQ$NaA2qgoV+9S>==faR{7FsS-*i?@(Ggmr{QqIqBO9rBi#P( zPGfmfdcDe*Mvw`NJ$T(P8p0z+pjfp0`c(i|`l!zY_*_kP7cK(h(pDAYrzOjdhmh1O zeCY|*0`o|m)R4mSp8Sp<_>TM;MF{Z;e7ROcxvLKF%fK<`IC}L_Kbp@aw;&Y>-G(2# ziueu6gz9R#g?$0*rhkoyz-$xO^`=w~{{qF?uLfP%+!y}XnWbgRnr{$<3pg|smVxK} z3+~Yfcg3*de2iS=lTvX#bK`F2la1)Ql~`171c zXa@h9LfzeE@8;hgrS4LzSKGri8Qs8gW)Mnq^;obp#^87*o0e(OUze>#zgt7Wp&dlS zXWs_W2ub0=Gu+6+H3w0r$b(k=o|U@Zhq-k;nAe;~wycW9thjpvBXjF`#N9ZKWE-n6 zk*i{m4Qj>URX&yy`7dWN+R-Zs~IX#Km0LOW461|HzMb65lUe5Em zl_A%$Hm1Zo2C@x7Zn!Fh-vm#WFKXpRcxPKWYBkO`8rfHO@=a`+}p)J zfS5fBkR9O;e9;hGillcG&=6r||58kuQw21V>*Q7PW(wkd4jM{{$!yt7A@X}xF4>4? zw9i3le0Ua+{^Ou*jwr%3^_c@<bYnazdc21Q1+O7cFd(req|o`e>9h7i6p+g3gc9jFl`>Ckku?9 z?Un7N9#X~-vRZiZ4CK0+^1cwjyh@mxT`_1}wl@T5Y7SYklgZ8NDG-+LMkh*}7ZUReikKeYgMZjr~~6)?{w zR?jyUP=A>QsTE}QZ;^4FSx7OO*pZ9KyX$@PG-g1-)OpR4mBWiMuXijWV|24G6z~S? zD9E2}|06}?D-NHLq5ltPJ3N>+W^_?*QSodZw~z+7*(aUxqR>q}{*CtehK^)kga6pSY? zD5ZEW`GN#^oC{t`v0i8y|Y7gG-r z&u=ZJ@eS8WT)h~^^;tqa^w`#=x6Eomo49-l+3MAgZnEj>+@u-7Z97!r21bdgOvD0O^j9R0()K{TLZG%D%!JtDMfo>`bcdO zE2d$nk6lV}QsT;`Xj0+JQ7&IDr7S~zAT7xpungnEa4%vR%14(` zV7-*MIPF4e!;4mcSLfw4OmcTDr*!!5$a0$AfNbL{)3L^=9j0Wp>cG+!^rS?;wSuy} zG^fRzwU*Mo3{IH55g#Q_VS@$Ip_MejOUuLfSlL{9@Ff>gXLkd{@QPJ5k@L%uSaqu? zw_b33g4S~&GfK9by{j9Qinld(pAear_tt>AsGRbo#Ea!nOG}jmv(|-rRlo-&Yp7E_ z_XM?Fb>-fOJFiyY1CO<2m((-YVopq42YzTxk2l~a@X594LdX+nlfY{fDrmT~X&p-X zldM*rw2o#;PG{E91k`2BdhJ-bo+j|=3NUuBpqcfdv!XYmS4F9uJIHB{&{-ce5AEEc zj5;MZ{kq9DUR_Pdl@Qj12MtuQhT=Ndv;wqRAAn`&lvUKRU(5d3tlia1vOSQ8K+fvr z`@Xa&QR_9e71UR6!%iY$NewU`9?N7gxR7nAK(*z(2B0g!Y!+Gc9oT0Bb$6u`T-;?2|tn?2J2qLeEe)PS>(j8#2^~_ z5>|;`TkyW{*f!iR*g}Kl#FtkWx6lCg*bZpoR?Pmw?dY&Ox6&9<O8c!8h* zowAKOP?ALhFW83eorEtMcA`t1-$v1tga}9i({{?j6i_u;v7JWor!V3awC^skF;-GP z^idVTIC=A zBKE2`cAz^}yh#1sEM}MDa86%dzY}nHB^7X&+Hi`IUw?^g-2Y`@|}rqEUGKDfVxW$jv)+2l^EU|rH|7J3dqvgEx)U+ ze39C9xqmGi^huha@kxov77MnA-Owd+HwF48YExuIm+!#~;0LGZ1m8GI!{xE@ z|1~(`JNkgzeFC(5`zWCPKQZ|DDf*BHoC7+*ZSj8>*oeQQ_xZ_Bfw|%wI5<9`v%Kyz zVBmO#k9>{`^Y>Un4}D7K`PzBcQR$lOC;z441h={Xkk@(O_P?k-ssBviqcNUw39tzl z=quiE89aY`1IplE19uHy%O4?b}Un%t-X0btti_^KI!UjpP;Lzg+} zD-htxhi~=3SHbG-#IGs6;wXmGxLANx9;>gaAU_WNj*j!+uYuh34$2o-9D&ULKQ+rOoAeC;MEar}|n{Tol_Th#E& zv5R+ltp@*w7uA8(_C8X6@*GI6z5~Xu-Nr3o@)uMeC!8N3__y0gY5XdDmP3EQEsobY z>yG5|J>B5vf7G6W_s?Eml=o}bXj?wUzl_7tRZj#O7ocA-u(*FX6Cjzkm delta 6264 zcmZu#d018Dx_`cRu{YD+u#rjTjbJDWY=*rVJOU41DoDF}SYmdq1Q4 z_2p^m_Md%JOQ)&gVHjrx1*n}ry0YaB5!xe=gn?@w7LTjrzwS_*g9B7+prJ1O$7DtQ z^&4p)E>F%;vx6*Z`2eG8x#`NKQLxbtmts&;)EVDNV;@b$`m>>~<+oF$zSTKRJsxXT zvj-XYdKD71{`rF%{zH4+ueaVHb^I?<>boWZ&f@dLC^c-TPE-rws=FW8$9YO^{gE3SV>Ae0r&tQAYkVK>W#1c#3u>Y}PCMxt&d6C&deS+Z71qbv$iD?@?s}DU2*GEd zZlYzqYM|IiydV@2<~{1ig+t)0Buk9s&Ou@q{}d+Ld7MpnbGZ{>hScD!juqn-Zi^Bw zJbi#DfSET}+k{!GyEN-2d%?ul;>99q(C&n*tqR8t64SUk9VNZ8i9Fd>3mCRM?Zcg$ z$jA|!MVQ)k2W8aDym2tfTsBNM@^GR&7c{6j;mAYq%mlcHc#guqB%?ajIl@hCE(C-! z^^7|BDLd*BQfziUIzoI+oRO}p-Quko5PUh;i@$tN#H*HhUi^LzjI{csjTU=|4Oys< z`Na@t(HMEyv}_zUWvsYQ;bA#qn$w&o1_`bjg<^pl;IvH;qli22gHilHBhiX(r z^NV_*fxPoY@e)TCi*}fXs0~ZH(Ppa(;nE3Wma~14_?YVdQKN+x?oDuJD#6z~25c5v9-ewVk0{`Vw-`A}D%Yyv+8cIC4ep z9Y?D9d%6R|JWq9QEfa@`qn4p8rhkQT!!~4cZKoE*4z61c>)<_i9G$->tl~8>+qrv< zm_a;!70xeNCkASpJ3q~ZZ>$v`^N+8~L8}%)oL3?AHP>|@eD^wW3Jv&g36dib+)#x; zySJNW=g;2j(UTsu1ghm}E^6M#AyU)Cxf$3_f2yLx3C9-|4qq$BZkq_!Yp(6KOD)|L z^54X@Zy@;2&m;P(tsXzwt)Qh|@~-%Z_*5k@4@W5&yA7H}usE0eRpH73h{0l6KGt()Cz_}$kSw}@QxBQ*FDKUwk zTts(s=`rCY+5MzW;S7r7h9=?1EsM#*t+U9B-EUyq+K!4BxM2oGO0CkF^Z?stQa|Qr zDTUjvh<=>+424LU?Q@aDd8bjt9yi{+!G(tRYWYj8rA;D>OOFbxBoJ5#k326%^v3i+ zt8n9@S=3*O3tPqiA5-=@k*qmu6?SfT2Juc^Nc}vXmPK%+%OcdaA(&kYDXzjo0lefQ z4sE!AuGPjZv2rerMNsT4Mx05eq7%sMsZE6uyERQ6v@~ABODx3NFMwK z$ckUYAfD0$M6A6aA}X$6k@O^LGJ3e%<|j_&;>#jK=DR>v%w2})UxeC!RYcKz;V02g z({tE4VJ7NQT1??m#daN&dA$flg~}l5)Qf8-ASuLYmFTmB2?_suY4#lw7EmWy15&eq@L#fIKBGlin ziyUrA#FOO|LTyVWTv04gDV|58x!gj*cjPs1&R_CsPDD?}`ctK>oRPa;MLFiqC2zj? zy@;wvf+^0qhbvD2U)z>Znk-!202;yBae5VN2M~JP4PdZsO$C*|P>6SQuy3pVL zs%7G^e^MFGD$7Q^j{sVhQM7asxeTgsajztbvyiHWg31;82|HF$ps@0h6*O8Z zc&*gPn3XzN2$H{B0VMHC9o#LFlii4(mtWRbbmA&->Klc>bMh+MNs7ZUm<#@k;+(_F zD8tN|m6Rot*{2*E?#)WNmxC$+t&no`-@67su2qB(Dlh}P|mho-a z*|?;F`g8u6xZj~X{&h2EGUQzviyC#VC0sytr=HC3Li_u7p!oY*#FV!m9>f;P#!G5k z17%pJ<4_RFNw=QaQW^)^>ibh9*n0sfH#XvdCJ>cY`x&p4x}FZM8JiRp+Ml zLGJfjN|v?>yKvu#sKqHS@1lp?yUi05rDiza-bJyLoX)Xz^sv-kS_gnF-UaQmbu_+L z+s=t~0KA6G2ap7cz2F0y*eg$!Kn>@c5OFi$E7ByRzr?F6uRo0Jdf;pBwMoB2CH4Tt zx~C-Tg|SI6hiKsb<6o54V~~{C$A4ls4F%|5+f4;B>Z`lyA(U%`lO{>B%1Pr^e+RaHsDcWi8yv_2l0(gBt$o56~m-_B5>@DX8RHjBn6>yfpIb$=Vx=HoOD* z{dyWg_H=Ho*R#5}0a@8JTf2>WrnDOeR~oSDk-E=8dK@U4eUN7L1ZwBVtr+o4%%3Z+ z znh%3?@H2<$1u4CBn4WaglXRr$;}~%iQ{40kCA#AhEJMIeQ=3x=hPykum9=$m9RWu-5eS>Rcp45{XU~yH@nW#`kKsDpV5Qe*{5>t z7A$pbp;K%TV-{vHB#i(RGZ5Q)(*P z&VYFAIOSjo7c}EJ?KE<4JWh#}nl7b5-1Y^zux}Hk0rZ;sHK}swBvygt4C&vK)-~a# z)qV{2^-XjSz^z$EG~?emXR-Ap-jRHo(z*O3cvny3ZBM5V zXhRc}C!V2Hfi}ugAw4hNcZTegu0<2frn7y~Akj&{wpOw0XYsx}d0LjI1+Rs3Ut;IC zwIGxOd-0xHbq+kl6U?>eaB*5K3x*^Ec(&9__<~}PVWjLDvhC(#5F}UYB^vY?pF9gI zq#Puz(!J7kP)tfs%1E@~g>W8*vFFLpoT`IwQmP{ZMbY>L=gId@eyT(Ahx}ibmA`}w z(&Yh8Lo4mMi%l-dC4Q5XG4iSV^RRcca)b{m%KbdWsC>;=Tp+!4k@WAgmYevV843kI zBNXBFfacb-BkESePbRr2K!RG^&R-PRIov!}Mw_F~m zKLzMtqr9Tw;e-CLdoxtgw!5dEd`FPZyT-vG%GBQPX6zD}iFktlxlId^R#NVm`uui`04ACm=1iPq=UZEUW{S?|4&` z{-jHt$Q*g>AEX3vbqpHeeiy_buk!F1Wz3qDh`VI~_|Fg0C@UA;rM$_73CdM+IGj(~ HlqC9JJ4|Y3 diff --git a/pkg/services/control/service.proto b/pkg/services/control/service.proto index e3b507387..34c040f2a 100644 --- a/pkg/services/control/service.proto +++ b/pkg/services/control/service.proto @@ -56,6 +56,9 @@ service ControlService { // Remove local access policy engine overrides stored in the node by chaind id. rpc RemoveChainLocalOverride (RemoveChainLocalOverrideRequest) returns (RemoveChainLocalOverrideResponse); + + // Flush objects from write-cache and move it to degraded read only mode. + rpc SealWriteCache(SealWriteCacheRequest) returns (SealWriteCacheResponse); } // Health check request. @@ -525,3 +528,32 @@ message RemoveChainLocalOverrideResponse { Signature signature = 2; } + +message SealWriteCacheRequest { + // Request body structure. + message Body { + // ID of the shard. + repeated bytes shard_ID = 1; + + // Flag indicating whether object read errors should be ignored. + bool ignore_errors = 2; + } + + Body body = 1; + Signature signature = 2; +} + +message SealWriteCacheResponse { + message Body { + message Status { + bytes shard_ID = 1; + bool success = 2; + string error = 3; + } + repeated Status results = 1; + } + + Body body = 1; + + Signature signature = 2; +} \ No newline at end of file diff --git a/pkg/services/control/service_frostfs.pb.go b/pkg/services/control/service_frostfs.pb.go index 3ace081e46092e07c6f512da4b6cb75c0cda0bf9..4ce6791f89f74cd6a09d14bd7e40ab8cb5d62d32 100644 GIT binary patch delta 490 zcmdn6k9G5B)`l&N2JzD$2r_o^2B#+GgcoI&q&g=iXH1@$89HgI-R3F#7BcdnNr+C4 zzbdymp|ORL6Rv0Sg;^q-npKz(BAYMFnu)}noHzFoniU0ch3rr(C!d{nS_y6#$N{NA zsl^5PdBv&mPWdU7@xdjDC8foa8Tmb??`CJ@n*1S?b#m@Lb@t%W2U5gH|wuCJPD)PrjGKH(4!Lm=$c;wloTseb8${KY}R0d@HgM}oQ)hZ^$RT~Cp6~Dq~#YW zWGYzMDikH=rKbYjimbgc+8U~Ta;?t-B>N^T5#8KpffBG2mdrv9SpDUX(E`>XqX-_b I6IM6_0MNd_=l}o! delta 11 Scmdn|nRUZH)`l&N2JrwQNd)5n diff --git a/pkg/services/control/service_grpc.pb.go b/pkg/services/control/service_grpc.pb.go index c3976c54aac5a708be939f5a4083731bf0aaa004..e09f6750b66eea396b4f5deb3be059fa191cf88a 100644 GIT binary patch delta 638 zcmezSjj^efX+s0YWLtIt&fwIxbAOHs(pFH2R(EKw-QS4c@sFG@^FO;IRH zO-xb9&&#O<%BH01aUpC2nLasDR2m^N*-=FkiFe;vZ1Y3Wbg{|z#TAjAgjZ9(iV7dx zLm(?sCo{?$p$j?4$8Pq}QR0Sq%!isDn=Ifcp@iy&pwzgi81YJC|WdGKSi2PA@*dK3Skr7|GD{73yft5UWlH0MilT A*#H0l delta 43 zcmZo_W%~P#aYF;g=Eod~yqh`2)5SI))KlW#++oGWzd6_IyUu3rssrqkb(+Eeg%J@B