From 4944490ffbe2dc16b8d28d06395cf9478de42165 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Mon, 18 Jul 2022 14:44:29 +0300 Subject: [PATCH] [#1559] local_object_storage: Move shard to the `DegradedReadOnly` mode `Degraded` mode can be set by the administrator if needed. Modifying operations in this mode can lead node into an inconsistent state because metabase checks such as lock checking are not performed. Signed-off-by: Evgenii Stratonikov --- .../modules/morph/internal/types.pb.go | Bin 6464 -> 6464 bytes cmd/neofs-cli/modules/control/shards_list.go | 2 ++ .../modules/control/shards_set_mode.go | 9 ++++--- cmd/neofs-node/config/engine/shard/config.go | 2 ++ pkg/local_object_storage/engine/engine.go | 2 +- pkg/local_object_storage/engine/error_test.go | 4 ++-- pkg/local_object_storage/metabase/mode.go | 8 +++---- pkg/local_object_storage/shard/get.go | 3 +-- pkg/local_object_storage/shard/mode/mode.go | 22 ++++++++++++------ pkg/services/control/ir/service.pb.go | Bin 14528 -> 14528 bytes pkg/services/control/ir/service_grpc.pb.go | Bin 3853 -> 3853 bytes pkg/services/control/ir/types.pb.go | Bin 7733 -> 7733 bytes pkg/services/control/server/list_shards.go | 2 ++ pkg/services/control/server/set_shard_mode.go | 2 ++ pkg/services/control/service.pb.go | Bin 100536 -> 100536 bytes pkg/services/control/service_grpc.pb.go | Bin 16663 -> 16663 bytes pkg/services/control/types.pb.go | Bin 26791 -> 27058 bytes pkg/services/control/types.proto | 3 +++ 18 files changed, 40 insertions(+), 19 deletions(-) diff --git a/cmd/neofs-adm/internal/modules/morph/internal/types.pb.go b/cmd/neofs-adm/internal/modules/morph/internal/types.pb.go index e76a29ae52de34d8387ff76d2adba2f62bf34e30..cdbe2045bdc37146eb69fa8ae0d23161e7403582 100644 GIT binary patch delta 15 WcmX?LbiimrE{l<&p3%m_^AZ3rdj+xp delta 15 WcmX?LbiimrE{mb1p2^0-^AZ3rwFSEX diff --git a/cmd/neofs-cli/modules/control/shards_list.go b/cmd/neofs-cli/modules/control/shards_list.go index 9ee986805..f6719d022 100644 --- a/cmd/neofs-cli/modules/control/shards_list.go +++ b/cmd/neofs-cli/modules/control/shards_list.go @@ -109,6 +109,8 @@ func shardModeToString(m control.ShardMode) string { return "read-only" case control.ShardMode_DEGRADED: return "degraded" + case control.ShardMode_DEGRADED_READ_ONLY: + return "degraded-read-only" default: return "unknown" } diff --git a/cmd/neofs-cli/modules/control/shards_set_mode.go b/cmd/neofs-cli/modules/control/shards_set_mode.go index 3c1eb6da9..625ec0434 100644 --- a/cmd/neofs-cli/modules/control/shards_set_mode.go +++ b/cmd/neofs-cli/modules/control/shards_set_mode.go @@ -17,9 +17,10 @@ const ( shardIDFlag = "id" shardClearErrorsFlag = "clear-errors" - shardModeReadOnly = "read-only" - shardModeReadWrite = "read-write" - shardModeDegraded = "degraded" + shardModeReadOnly = "read-only" + shardModeReadWrite = "read-write" + shardModeDegraded = "degraded" + shardModeDegradedReadOnly = "degraded-read-only" ) var setShardModeCmd = &cobra.Command{ @@ -60,6 +61,8 @@ func setShardMode(cmd *cobra.Command, _ []string) { mode = control.ShardMode_READ_ONLY case shardModeDegraded: mode = control.ShardMode_DEGRADED + case shardModeDegradedReadOnly: + mode = control.ShardMode_DEGRADED_READ_ONLY } req := new(control.SetShardModeRequest) diff --git a/cmd/neofs-node/config/engine/shard/config.go b/cmd/neofs-node/config/engine/shard/config.go index 0bb8b2112..77c032a53 100644 --- a/cmd/neofs-node/config/engine/shard/config.go +++ b/cmd/neofs-node/config/engine/shard/config.go @@ -88,6 +88,8 @@ func (x *Config) Mode() (m mode.Mode) { m = mode.ReadOnly case "degraded": m = mode.Degraded + case "degraded-read-only": + m = mode.DegradedReadOnly default: panic(fmt.Sprintf("unknown shard mode: %s", s)) } diff --git a/pkg/local_object_storage/engine/engine.go b/pkg/local_object_storage/engine/engine.go index 115cb7cd2..64041f623 100644 --- a/pkg/local_object_storage/engine/engine.go +++ b/pkg/local_object_storage/engine/engine.go @@ -51,7 +51,7 @@ func (e *StorageEngine) reportShardError( return } - err = sh.SetMode(mode.Degraded) + err = sh.SetMode(mode.DegradedReadOnly) if err != nil { e.log.Error("failed to move shard in degraded mode", zap.Uint32("error count", errCount), diff --git a/pkg/local_object_storage/engine/error_test.go b/pkg/local_object_storage/engine/error_test.go index 9f049132d..c2907f3c5 100644 --- a/pkg/local_object_storage/engine/error_test.go +++ b/pkg/local_object_storage/engine/error_test.go @@ -125,7 +125,7 @@ func TestErrorReporting(t *testing.T) { for i := uint32(0); i < 2; i++ { _, err = e.Get(GetPrm{addr: object.AddressOf(obj)}) require.Error(t, err) - checkShardState(t, e, id[0], errThreshold+i, mode.Degraded) + checkShardState(t, e, id[0], errThreshold+i, mode.DegradedReadOnly) checkShardState(t, e, id[1], 0, mode.ReadWrite) } @@ -193,7 +193,7 @@ func TestBlobstorFailback(t *testing.T) { require.ErrorAs(t, err, &apistatus.ObjectOutOfRange{}) } - checkShardState(t, e, id[0], 1, mode.Degraded) + checkShardState(t, e, id[0], 1, mode.DegradedReadOnly) checkShardState(t, e, id[1], 0, mode.ReadWrite) } diff --git a/pkg/local_object_storage/metabase/mode.go b/pkg/local_object_storage/metabase/mode.go index 9267bff3f..f5d4a8433 100644 --- a/pkg/local_object_storage/metabase/mode.go +++ b/pkg/local_object_storage/metabase/mode.go @@ -23,12 +23,12 @@ func (db *DB) SetMode(m mode.Mode) error { } var err error - switch m { - case mode.Degraded: + switch { + case m.NoMetabase(): db.boltDB = nil - case mode.ReadOnly: + case m.ReadOnly(): err = db.Open(true) - case mode.ReadWrite: + default: err = db.Open(false) } if err == nil && !m.NoMetabase() && !m.ReadOnly() { diff --git a/pkg/local_object_storage/shard/get.go b/pkg/local_object_storage/shard/get.go index 16bd54c02..2647e74d9 100644 --- a/pkg/local_object_storage/shard/get.go +++ b/pkg/local_object_storage/shard/get.go @@ -6,7 +6,6 @@ import ( "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobovnicza" "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor" meta "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/metabase" - "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/shard/mode" "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/writecache" apistatus "github.com/nspcc-dev/neofs-sdk-go/client/status" objectSDK "github.com/nspcc-dev/neofs-sdk-go/object" @@ -123,7 +122,7 @@ func (s *Shard) fetchObjectData(addr oid.Address, skipMeta bool, big, small stor mPrm.SetAddress(addr) mRes, err := s.metaBase.Exists(mPrm) - if err != nil && s.GetMode() != mode.Degraded { + if err != nil && !s.GetMode().NoMetabase() { return res, false, err } exists = mRes.Exists() diff --git a/pkg/local_object_storage/shard/mode/mode.go b/pkg/local_object_storage/shard/mode/mode.go index a0aef6e61..a22963612 100644 --- a/pkg/local_object_storage/shard/mode/mode.go +++ b/pkg/local_object_storage/shard/mode/mode.go @@ -3,19 +3,25 @@ package mode // Mode represents enumeration of Shard work modes. type Mode uint32 -// ReadWrite is a Mode value for shard that is available -// for read and write operations. Default shard mode. -const ReadWrite Mode = 0 +const ( + // ReadWrite is a Mode value for shard that is available + // for read and write operations. Default shard mode. + ReadWrite Mode = 0 + + // DegradedReadOnly is a Mode value for shard that is set automatically + // after a certain number of errors is encountered. It is the same as + // `mode.ReadOnly` but also enables fallback algorithms for getting object + // in case metabase is corrupted. + DegradedReadOnly = Degraded | ReadOnly +) const ( // ReadOnly is a Mode value for shard that does not // accept write operation but is readable. ReadOnly Mode = 1 << iota - // Degraded is a Mode value for shard that is set automatically - // after a certain number of errors is encountered. It is the same as - // `mode.ReadOnly` but also enables fallback algorithms for getting object - // in case metabase is corrupted. + // Degraded is a Mode value for shard when the metabase is unavailable. + // It is hard to perform some modifying operations in this mode, thus it can only be set by an administrator. Degraded ) @@ -29,6 +35,8 @@ func (m Mode) String() string { return "READ_ONLY" case Degraded: return "DEGRADED" + case DegradedReadOnly: + return "DEGRADED_READ_ONLY" } } diff --git a/pkg/services/control/ir/service.pb.go b/pkg/services/control/ir/service.pb.go index bd6d26de962959588ddde0096fe053c6c3b2ba10..61b58068e681716a55c200ca88352b225e564d52 100644 GIT binary patch delta 12 TcmX?5c%X1X0i)5zLVpVYCfx-; delta 12 TcmX?5c%X1X0i)r@LVpVYCfEf& diff --git a/pkg/services/control/ir/service_grpc.pb.go b/pkg/services/control/ir/service_grpc.pb.go index edd84812717b57265f753eb0d1ab93f4bd26729d..9ff723e6335027cac5092dfb17d367784e164890 100644 GIT binary patch delta 12 TcmeB`>y?|(z-Y9wv4;-;9Pb2X delta 12 TcmeB`>y?|(z-YL!v4;-;9O?vR diff --git a/pkg/services/control/ir/types.pb.go b/pkg/services/control/ir/types.pb.go index 13f4499ed6097b7223c3b3c7b8cd4518f3d1ff98..289f09cda6d324bb0d715ed1e6e3857fc0ea6e7f 100644 GIT binary patch delta 12 TcmdmLv(;uo0i)5z!XvT(B9;Xi delta 12 TcmdmLv(;uo0i)r@!XvT(B9R3c diff --git a/pkg/services/control/server/list_shards.go b/pkg/services/control/server/list_shards.go index 8e6ac1b60..11a66dcd5 100644 --- a/pkg/services/control/server/list_shards.go +++ b/pkg/services/control/server/list_shards.go @@ -43,6 +43,8 @@ func (s *Server) ListShards(_ context.Context, req *control.ListShardsRequest) ( m = control.ShardMode_READ_ONLY case mode.Degraded: m = control.ShardMode_DEGRADED + case mode.DegradedReadOnly: + m = control.ShardMode_DEGRADED_READ_ONLY default: m = control.ShardMode_SHARD_MODE_UNDEFINED } diff --git a/pkg/services/control/server/set_shard_mode.go b/pkg/services/control/server/set_shard_mode.go index 46ecdd6c5..94bf438c6 100644 --- a/pkg/services/control/server/set_shard_mode.go +++ b/pkg/services/control/server/set_shard_mode.go @@ -32,6 +32,8 @@ func (s *Server) SetShardMode(_ context.Context, req *control.SetShardModeReques m = mode.ReadOnly case control.ShardMode_DEGRADED: m = mode.Degraded + case control.ShardMode_DEGRADED_READ_ONLY: + m = mode.ReadOnly default: return nil, status.Error(codes.Internal, fmt.Sprintf("unknown shard mode: %s", requestedMode)) } diff --git a/pkg/services/control/service.pb.go b/pkg/services/control/service.pb.go index 4ad323c075fe13b25b1fad8127fa8fce8c6a852c..8bc8c282ee8e1cae7ef368e468bf7951429218c8 100644 GIT binary patch delta 17 Ycmdlnk!{CBwh0A{MvaAA3mM%z06-)Lx&QzG delta 17 Ycmdlnk!{CBwh0A{hK+?=3mM%z06-iDxc~qF diff --git a/pkg/services/control/service_grpc.pb.go b/pkg/services/control/service_grpc.pb.go index bee17aee507a10d8da54944a756470b9b1725a59..8632c47822cf1f0312dd5bd018b855e5d6ad92af 100644 GIT binary patch delta 14 VcmbQ<#5lc)aY6&5(Zk4ge`W1%Chl diff --git a/pkg/services/control/types.pb.go b/pkg/services/control/types.pb.go index c22eb0a7e362803deb41ad3485625c9277c53cd9..4c3d3c9ca7e1b86292d33e6ca4ec4b008eb27269 100644 GIT binary patch delta 299 zcmZ2}k#W;y#t8+CMjH#e_#8R)^%Y!F(~A;QQd5Fb6I1;2aw_$>ID<11i&A{^Q&Qty zT-}2lU0hw_gIpb5;{E-6A{F403bqO+ljm_tPZs6x6z4RtQc%KFJ$V-Y0x3=`5=vGI z3JNBh%?0+jFqtK8{u=U?QN++lN5P=N&jANQM z*)L3zQDJgsn7SB58f<_u*Z|W+pwh{|!t^JLdkIXo3RhqPa-+lbf$Dq0^+B9H;rf%8 cIB0PI%{5B{DVBFtm~0!t·FCtO^0PYS|#{d8T delta 98 zcmdmVnQ{3=#t8+Ch8qjJ_$D7^mYN*S-#Pg=|D4TR1a`VGnHp`*5B;UItj diff --git a/pkg/services/control/types.proto b/pkg/services/control/types.proto index f9c8cef2c..0729e4ebb 100644 --- a/pkg/services/control/types.proto +++ b/pkg/services/control/types.proto @@ -157,4 +157,7 @@ enum ShardMode { // Degraded. DEGRADED = 3; + + // DegradedReadOnly. + DEGRADED_READ_ONLY = 4; }