diff --git a/cmd/neofs-cli/modules/control.go b/cmd/neofs-cli/modules/control.go index 9a0e0aa251..bf02668844 100644 --- a/cmd/neofs-cli/modules/control.go +++ b/cmd/neofs-cli/modules/control.go @@ -78,6 +78,7 @@ const ( shardModeReadOnly = "read-only" shardModeReadWrite = "read-write" + shardModeDegraded = "degraded" ) const ( @@ -124,9 +125,10 @@ func initControlSetShardModeCmd() { flags.String(controlRPC, controlRPCDefault, controlRPCUsage) flags.StringVarP(&shardID, shardIDFlag, "", "", "ID of the shard in base58 encoding") flags.StringVarP(&shardMode, shardModeFlag, "", "", - fmt.Sprintf("new shard mode keyword ('%s', '%s')", + fmt.Sprintf("new shard mode keyword ('%s', '%s', '%s')", shardModeReadWrite, shardModeReadOnly, + shardModeDegraded, ), ) flags.Bool(shardClearErrorsFlag, false, "Set shard error count to 0") @@ -490,6 +492,8 @@ func prettyPrintShards(cmd *cobra.Command, ii []*control.ShardInfo) { mode = "read-write" case control.ShardMode_READ_ONLY: mode = "read-only" + case control.ShardMode_DEGRADED: + mode = "degraded" default: mode = "unknown" } @@ -526,6 +530,8 @@ func setShardMode(cmd *cobra.Command, _ []string) { mode = control.ShardMode_READ_WRITE case shardModeReadOnly: mode = control.ShardMode_READ_ONLY + case shardModeDegraded: + mode = control.ShardMode_DEGRADED } req := new(control.SetShardModeRequest) diff --git a/cmd/neofs-node/config/engine/shard/config.go b/cmd/neofs-node/config/engine/shard/config.go index 147b1c3374..bb395afd6a 100644 --- a/cmd/neofs-node/config/engine/shard/config.go +++ b/cmd/neofs-node/config/engine/shard/config.go @@ -77,6 +77,8 @@ func (x *Config) Mode() (m shard.Mode) { m = shard.ModeReadWrite case "read-only": m = shard.ModeReadOnly + case "degraded": + m = shard.ModeDegraded default: panic(fmt.Sprintf("unknown shard mode: %s", s)) } diff --git a/pkg/local_object_storage/engine/engine.go b/pkg/local_object_storage/engine/engine.go index 24fb662068..f40f158c5b 100644 --- a/pkg/local_object_storage/engine/engine.go +++ b/pkg/local_object_storage/engine/engine.go @@ -50,13 +50,13 @@ func (e *StorageEngine) reportShardError( return } - err = sh.SetMode(shard.ModeReadOnly) + err = sh.SetMode(shard.ModeDegraded) if err != nil { - e.log.Error("failed to move shard in read-only mode", + e.log.Error("failed to move shard in degraded mode", zap.Uint32("error count", errCount), zap.Error(err)) } else { - e.log.Info("shard is moved in read-only due to error threshold", + e.log.Info("shard is moved in degraded mode due to error threshold", zap.Stringer("shard_id", sh.ID()), zap.Uint32("error count", errCount)) } diff --git a/pkg/local_object_storage/engine/error_test.go b/pkg/local_object_storage/engine/error_test.go index 7e943e9e07..7fe345e664 100644 --- a/pkg/local_object_storage/engine/error_test.go +++ b/pkg/local_object_storage/engine/error_test.go @@ -118,7 +118,7 @@ func TestErrorReporting(t *testing.T) { for i := uint32(0); i < 2; i++ { _, err = e.Get(&GetPrm{addr: object.AddressOf(obj)}) require.Error(t, err) - checkShardState(t, e, id[0], errThreshold+i, shard.ModeReadOnly) + checkShardState(t, e, id[0], errThreshold+i, shard.ModeDegraded) checkShardState(t, e, id[1], 0, shard.ModeReadWrite) } @@ -185,7 +185,7 @@ func TestBlobstorFailback(t *testing.T) { require.True(t, errors.Is(err, object.ErrRangeOutOfBounds), "got: %v", err) } - checkShardState(t, e, id[0], 4, shard.ModeReadOnly) + checkShardState(t, e, id[0], 4, shard.ModeDegraded) checkShardState(t, e, id[1], 0, shard.ModeReadWrite) } diff --git a/pkg/local_object_storage/shard/delete.go b/pkg/local_object_storage/shard/delete.go index 1b4e33b996..76499d4b35 100644 --- a/pkg/local_object_storage/shard/delete.go +++ b/pkg/local_object_storage/shard/delete.go @@ -31,7 +31,7 @@ func (p *DeletePrm) WithAddresses(addr ...*addressSDK.Address) *DeletePrm { // Delete removes data from the shard's writeCache, metaBase and // blobStor. func (s *Shard) Delete(prm *DeletePrm) (*DeleteRes, error) { - if s.GetMode() == ModeReadOnly { + if s.GetMode() != ModeReadWrite { return nil, ErrReadOnlyMode } diff --git a/pkg/local_object_storage/shard/gc.go b/pkg/local_object_storage/shard/gc.go index 1e7dfce707..57b84b8a85 100644 --- a/pkg/local_object_storage/shard/gc.go +++ b/pkg/local_object_storage/shard/gc.go @@ -174,7 +174,7 @@ func (gc *gc) stop() { // with GC-marked graves. // Does nothing if shard is in "read-only" mode. func (s *Shard) removeGarbage() { - if s.GetMode() == ModeReadOnly { + if s.GetMode() != ModeReadWrite { return } diff --git a/pkg/local_object_storage/shard/inhume.go b/pkg/local_object_storage/shard/inhume.go index e38c564ae0..2b4247cfc5 100644 --- a/pkg/local_object_storage/shard/inhume.go +++ b/pkg/local_object_storage/shard/inhume.go @@ -51,7 +51,7 @@ func (p *InhumePrm) MarkAsGarbage(addr ...*addressSDK.Address) *InhumePrm { // // Returns ErrReadOnlyMode error if shard is in "read-only" mode. func (s *Shard) Inhume(prm *InhumePrm) (*InhumeRes, error) { - if s.GetMode() == ModeReadOnly { + if s.GetMode() != ModeReadWrite { return nil, ErrReadOnlyMode } diff --git a/pkg/local_object_storage/shard/lock.go b/pkg/local_object_storage/shard/lock.go index b80462110f..916b38b3aa 100644 --- a/pkg/local_object_storage/shard/lock.go +++ b/pkg/local_object_storage/shard/lock.go @@ -14,7 +14,7 @@ import ( // // Locked list should be unique. Panics if it is empty. func (s *Shard) Lock(idCnr cid.ID, locker oid.ID, locked []oid.ID) error { - if s.GetMode() == ModeReadOnly { + if s.GetMode() != ModeReadWrite { return ErrReadOnlyMode } diff --git a/pkg/local_object_storage/shard/mode.go b/pkg/local_object_storage/shard/mode.go index 9a459eec76..1a6e85bc4f 100644 --- a/pkg/local_object_storage/shard/mode.go +++ b/pkg/local_object_storage/shard/mode.go @@ -21,6 +21,12 @@ const ( // ModeReadOnly is a Mode value for shard that does not // accept write operation but is readable. ModeReadOnly + + // ModeDegraded is a Mode value for shard that is set automatically + // after a certain number of errors is encountered. It is the same as + // `ModeReadOnly` but also enables fallback algorithms for getting object + // in case metabase is corrupted. + ModeDegraded ) func (m Mode) String() string { @@ -31,6 +37,8 @@ func (m Mode) String() string { return "READ_WRITE" case ModeReadOnly: return "READ_ONLY" + case ModeDegraded: + return "DEGRADED" } } @@ -46,6 +54,8 @@ func (s *Shard) SetMode(m Mode) error { switch m { case ModeReadOnly: s.writeCache.SetMode(writecache.ModeReadOnly) + case ModeDegraded: + s.writeCache.SetMode(writecache.ModeDegraded) case ModeReadWrite: s.writeCache.SetMode(writecache.ModeReadWrite) } diff --git a/pkg/local_object_storage/shard/move.go b/pkg/local_object_storage/shard/move.go index caf01f97be..826adbbf06 100644 --- a/pkg/local_object_storage/shard/move.go +++ b/pkg/local_object_storage/shard/move.go @@ -27,7 +27,7 @@ func (p *ToMoveItPrm) WithAddress(addr *addressSDK.Address) *ToMoveItPrm { // ToMoveIt calls metabase.ToMoveIt method to mark object as relocatable to // another shard. func (s *Shard) ToMoveIt(prm *ToMoveItPrm) (*ToMoveItRes, error) { - if s.GetMode() == ModeReadOnly { + if s.GetMode() != ModeReadWrite { return nil, ErrReadOnlyMode } diff --git a/pkg/local_object_storage/shard/put.go b/pkg/local_object_storage/shard/put.go index 4fab4616e2..f7f50d7ce4 100644 --- a/pkg/local_object_storage/shard/put.go +++ b/pkg/local_object_storage/shard/put.go @@ -33,7 +33,7 @@ func (p *PutPrm) WithObject(obj *object.Object) *PutPrm { // // Returns ErrReadOnlyMode error if shard is in "read-only" mode. func (s *Shard) Put(prm *PutPrm) (*PutRes, error) { - if s.GetMode() == ModeReadOnly { + if s.GetMode() != ModeReadWrite { return nil, ErrReadOnlyMode } diff --git a/pkg/local_object_storage/writecache/delete.go b/pkg/local_object_storage/writecache/delete.go index bfdeb46db3..3a90f1b51a 100644 --- a/pkg/local_object_storage/writecache/delete.go +++ b/pkg/local_object_storage/writecache/delete.go @@ -16,7 +16,7 @@ import ( func (c *cache) Delete(addr *addressSDK.Address) error { c.modeMtx.RLock() defer c.modeMtx.RUnlock() - if c.mode == ModeReadOnly { + if c.readOnly() { return ErrReadOnly } diff --git a/pkg/local_object_storage/writecache/flush.go b/pkg/local_object_storage/writecache/flush.go index d7f45c3664..c66a13bdd4 100644 --- a/pkg/local_object_storage/writecache/flush.go +++ b/pkg/local_object_storage/writecache/flush.go @@ -65,7 +65,7 @@ func (c *cache) flush() { sz := 0 c.modeMtx.RLock() - if c.mode == ModeReadOnly { + if c.readOnly() { c.modeMtx.RUnlock() time.Sleep(time.Second) continue @@ -127,7 +127,7 @@ func (c *cache) flushBigObjects() { select { case <-tick.C: c.modeMtx.RLock() - if c.mode == ModeReadOnly { + if c.readOnly() { c.modeMtx.RUnlock() break } diff --git a/pkg/local_object_storage/writecache/iterate.go b/pkg/local_object_storage/writecache/iterate.go index 596db0deb2..3c4af31bcb 100644 --- a/pkg/local_object_storage/writecache/iterate.go +++ b/pkg/local_object_storage/writecache/iterate.go @@ -36,7 +36,7 @@ func (p *IterationPrm) WithIgnoreErrors(ignore bool) *IterationPrm { func (c *cache) Iterate(prm *IterationPrm) error { c.modeMtx.RLock() defer c.modeMtx.RUnlock() - if c.mode != ModeReadOnly { + if !c.readOnly() { return nil } diff --git a/pkg/local_object_storage/writecache/mode.go b/pkg/local_object_storage/writecache/mode.go index fce239f5ab..8abd32d1dd 100644 --- a/pkg/local_object_storage/writecache/mode.go +++ b/pkg/local_object_storage/writecache/mode.go @@ -14,6 +14,9 @@ const ( // ModeReadOnly is a mode in which write-cache doesn't flush anything to a metabase. ModeReadOnly + + // ModeDegraded is similar to a shard's degraded mode. + ModeDegraded ) // ErrReadOnly is returned when Put/Write is performed in a read-only mode. @@ -50,3 +53,9 @@ func (c *cache) SetMode(m Mode) { time.Sleep(time.Second) } } + +// readOnly returns true if current mode is read-only. +// `c.modeMtx` must be taken. +func (c *cache) readOnly() bool { + return c.mode != ModeReadWrite +} diff --git a/pkg/local_object_storage/writecache/persist.go b/pkg/local_object_storage/writecache/persist.go index bf638c3537..7dfc6c86a4 100644 --- a/pkg/local_object_storage/writecache/persist.go +++ b/pkg/local_object_storage/writecache/persist.go @@ -21,7 +21,7 @@ func (c *cache) persistLoop() { select { case <-tick.C: c.modeMtx.RLock() - if c.mode == ModeReadOnly { + if c.readOnly() { c.modeMtx.RUnlock() continue } diff --git a/pkg/local_object_storage/writecache/put.go b/pkg/local_object_storage/writecache/put.go index 4c60b5b89f..1f4e6b5156 100644 --- a/pkg/local_object_storage/writecache/put.go +++ b/pkg/local_object_storage/writecache/put.go @@ -15,7 +15,7 @@ var ErrBigObject = errors.New("too big object") func (c *cache) Put(o *objectSDK.Object) error { c.modeMtx.RLock() defer c.modeMtx.RUnlock() - if c.mode == ModeReadOnly { + if c.readOnly() { return ErrReadOnly } diff --git a/pkg/services/control/server/list_shards.go b/pkg/services/control/server/list_shards.go index b02a45633a..3cc89284b7 100644 --- a/pkg/services/control/server/list_shards.go +++ b/pkg/services/control/server/list_shards.go @@ -40,6 +40,8 @@ func (s *Server) ListShards(_ context.Context, req *control.ListShardsRequest) ( mode = control.ShardMode_READ_WRITE case shard.ModeReadOnly: mode = control.ShardMode_READ_ONLY + case shard.ModeDegraded: + mode = control.ShardMode_DEGRADED default: mode = control.ShardMode_SHARD_MODE_UNDEFINED } diff --git a/pkg/services/control/server/set_shard_mode.go b/pkg/services/control/server/set_shard_mode.go index 01edcfdbda..9e3b7308cf 100644 --- a/pkg/services/control/server/set_shard_mode.go +++ b/pkg/services/control/server/set_shard_mode.go @@ -29,6 +29,8 @@ func (s *Server) SetShardMode(_ context.Context, req *control.SetShardModeReques mode = shard.ModeReadWrite case control.ShardMode_READ_ONLY: mode = shard.ModeReadOnly + case control.ShardMode_DEGRADED: + mode = shard.ModeDegraded default: return nil, status.Error(codes.Internal, fmt.Sprintf("unknown shard mode: %s", requestedMode)) } diff --git a/pkg/services/control/types.pb.go b/pkg/services/control/types.pb.go index 3674b0c782..16a8b75b5f 100644 Binary files a/pkg/services/control/types.pb.go and b/pkg/services/control/types.pb.go differ diff --git a/pkg/services/control/types.proto b/pkg/services/control/types.proto index bbc34c1791..a786c7b12f 100644 --- a/pkg/services/control/types.proto +++ b/pkg/services/control/types.proto @@ -151,4 +151,7 @@ enum ShardMode { // Read-only. READ_ONLY = 2; + + // Degraded. + DEGRADED = 3; }