diff --git a/cmd/neofs-adm/internal/modules/morph/internal/types.pb.go b/cmd/neofs-adm/internal/modules/morph/internal/types.pb.go index e76a29ae5..cdbe2045b 100644 Binary files a/cmd/neofs-adm/internal/modules/morph/internal/types.pb.go and b/cmd/neofs-adm/internal/modules/morph/internal/types.pb.go differ diff --git a/cmd/neofs-cli/modules/control/shards_list.go b/cmd/neofs-cli/modules/control/shards_list.go index 9ee986805..f6719d022 100644 --- a/cmd/neofs-cli/modules/control/shards_list.go +++ b/cmd/neofs-cli/modules/control/shards_list.go @@ -109,6 +109,8 @@ func shardModeToString(m control.ShardMode) string { return "read-only" case control.ShardMode_DEGRADED: return "degraded" + case control.ShardMode_DEGRADED_READ_ONLY: + return "degraded-read-only" default: return "unknown" } diff --git a/cmd/neofs-cli/modules/control/shards_set_mode.go b/cmd/neofs-cli/modules/control/shards_set_mode.go index 3c1eb6da9..625ec0434 100644 --- a/cmd/neofs-cli/modules/control/shards_set_mode.go +++ b/cmd/neofs-cli/modules/control/shards_set_mode.go @@ -17,9 +17,10 @@ const ( shardIDFlag = "id" shardClearErrorsFlag = "clear-errors" - shardModeReadOnly = "read-only" - shardModeReadWrite = "read-write" - shardModeDegraded = "degraded" + shardModeReadOnly = "read-only" + shardModeReadWrite = "read-write" + shardModeDegraded = "degraded" + shardModeDegradedReadOnly = "degraded-read-only" ) var setShardModeCmd = &cobra.Command{ @@ -60,6 +61,8 @@ func setShardMode(cmd *cobra.Command, _ []string) { mode = control.ShardMode_READ_ONLY case shardModeDegraded: mode = control.ShardMode_DEGRADED + case shardModeDegradedReadOnly: + mode = control.ShardMode_DEGRADED_READ_ONLY } req := new(control.SetShardModeRequest) diff --git a/cmd/neofs-node/config/engine/shard/config.go b/cmd/neofs-node/config/engine/shard/config.go index 0bb8b2112..77c032a53 100644 --- a/cmd/neofs-node/config/engine/shard/config.go +++ b/cmd/neofs-node/config/engine/shard/config.go @@ -88,6 +88,8 @@ func (x *Config) Mode() (m mode.Mode) { m = mode.ReadOnly case "degraded": m = mode.Degraded + case "degraded-read-only": + m = mode.DegradedReadOnly default: panic(fmt.Sprintf("unknown shard mode: %s", s)) } diff --git a/pkg/local_object_storage/engine/engine.go b/pkg/local_object_storage/engine/engine.go index 115cb7cd2..64041f623 100644 --- a/pkg/local_object_storage/engine/engine.go +++ b/pkg/local_object_storage/engine/engine.go @@ -51,7 +51,7 @@ func (e *StorageEngine) reportShardError( return } - err = sh.SetMode(mode.Degraded) + err = sh.SetMode(mode.DegradedReadOnly) if err != nil { e.log.Error("failed to move shard in degraded mode", zap.Uint32("error count", errCount), diff --git a/pkg/local_object_storage/engine/error_test.go b/pkg/local_object_storage/engine/error_test.go index 9f049132d..c2907f3c5 100644 --- a/pkg/local_object_storage/engine/error_test.go +++ b/pkg/local_object_storage/engine/error_test.go @@ -125,7 +125,7 @@ func TestErrorReporting(t *testing.T) { for i := uint32(0); i < 2; i++ { _, err = e.Get(GetPrm{addr: object.AddressOf(obj)}) require.Error(t, err) - checkShardState(t, e, id[0], errThreshold+i, mode.Degraded) + checkShardState(t, e, id[0], errThreshold+i, mode.DegradedReadOnly) checkShardState(t, e, id[1], 0, mode.ReadWrite) } @@ -193,7 +193,7 @@ func TestBlobstorFailback(t *testing.T) { require.ErrorAs(t, err, &apistatus.ObjectOutOfRange{}) } - checkShardState(t, e, id[0], 1, mode.Degraded) + checkShardState(t, e, id[0], 1, mode.DegradedReadOnly) checkShardState(t, e, id[1], 0, mode.ReadWrite) } diff --git a/pkg/local_object_storage/metabase/mode.go b/pkg/local_object_storage/metabase/mode.go index 9267bff3f..f5d4a8433 100644 --- a/pkg/local_object_storage/metabase/mode.go +++ b/pkg/local_object_storage/metabase/mode.go @@ -23,12 +23,12 @@ func (db *DB) SetMode(m mode.Mode) error { } var err error - switch m { - case mode.Degraded: + switch { + case m.NoMetabase(): db.boltDB = nil - case mode.ReadOnly: + case m.ReadOnly(): err = db.Open(true) - case mode.ReadWrite: + default: err = db.Open(false) } if err == nil && !m.NoMetabase() && !m.ReadOnly() { diff --git a/pkg/local_object_storage/shard/get.go b/pkg/local_object_storage/shard/get.go index 16bd54c02..2647e74d9 100644 --- a/pkg/local_object_storage/shard/get.go +++ b/pkg/local_object_storage/shard/get.go @@ -6,7 +6,6 @@ import ( "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobovnicza" "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor" meta "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/metabase" - "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/shard/mode" "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/writecache" apistatus "github.com/nspcc-dev/neofs-sdk-go/client/status" objectSDK "github.com/nspcc-dev/neofs-sdk-go/object" @@ -123,7 +122,7 @@ func (s *Shard) fetchObjectData(addr oid.Address, skipMeta bool, big, small stor mPrm.SetAddress(addr) mRes, err := s.metaBase.Exists(mPrm) - if err != nil && s.GetMode() != mode.Degraded { + if err != nil && !s.GetMode().NoMetabase() { return res, false, err } exists = mRes.Exists() diff --git a/pkg/local_object_storage/shard/mode/mode.go b/pkg/local_object_storage/shard/mode/mode.go index a0aef6e61..a22963612 100644 --- a/pkg/local_object_storage/shard/mode/mode.go +++ b/pkg/local_object_storage/shard/mode/mode.go @@ -3,19 +3,25 @@ package mode // Mode represents enumeration of Shard work modes. type Mode uint32 -// ReadWrite is a Mode value for shard that is available -// for read and write operations. Default shard mode. -const ReadWrite Mode = 0 +const ( + // ReadWrite is a Mode value for shard that is available + // for read and write operations. Default shard mode. + ReadWrite Mode = 0 + + // DegradedReadOnly is a Mode value for shard that is set automatically + // after a certain number of errors is encountered. It is the same as + // `mode.ReadOnly` but also enables fallback algorithms for getting object + // in case metabase is corrupted. + DegradedReadOnly = Degraded | ReadOnly +) const ( // ReadOnly is a Mode value for shard that does not // accept write operation but is readable. ReadOnly Mode = 1 << iota - // Degraded is a Mode value for shard that is set automatically - // after a certain number of errors is encountered. It is the same as - // `mode.ReadOnly` but also enables fallback algorithms for getting object - // in case metabase is corrupted. + // Degraded is a Mode value for shard when the metabase is unavailable. + // It is hard to perform some modifying operations in this mode, thus it can only be set by an administrator. Degraded ) @@ -29,6 +35,8 @@ func (m Mode) String() string { return "READ_ONLY" case Degraded: return "DEGRADED" + case DegradedReadOnly: + return "DEGRADED_READ_ONLY" } } diff --git a/pkg/services/control/ir/service.pb.go b/pkg/services/control/ir/service.pb.go index bd6d26de9..61b58068e 100644 Binary files a/pkg/services/control/ir/service.pb.go and b/pkg/services/control/ir/service.pb.go differ diff --git a/pkg/services/control/ir/service_grpc.pb.go b/pkg/services/control/ir/service_grpc.pb.go index edd848127..9ff723e63 100644 Binary files a/pkg/services/control/ir/service_grpc.pb.go and b/pkg/services/control/ir/service_grpc.pb.go differ diff --git a/pkg/services/control/ir/types.pb.go b/pkg/services/control/ir/types.pb.go index 13f4499ed..289f09cda 100644 Binary files a/pkg/services/control/ir/types.pb.go and b/pkg/services/control/ir/types.pb.go differ diff --git a/pkg/services/control/server/list_shards.go b/pkg/services/control/server/list_shards.go index 8e6ac1b60..11a66dcd5 100644 --- a/pkg/services/control/server/list_shards.go +++ b/pkg/services/control/server/list_shards.go @@ -43,6 +43,8 @@ func (s *Server) ListShards(_ context.Context, req *control.ListShardsRequest) ( m = control.ShardMode_READ_ONLY case mode.Degraded: m = control.ShardMode_DEGRADED + case mode.DegradedReadOnly: + m = control.ShardMode_DEGRADED_READ_ONLY default: m = control.ShardMode_SHARD_MODE_UNDEFINED } diff --git a/pkg/services/control/server/set_shard_mode.go b/pkg/services/control/server/set_shard_mode.go index 46ecdd6c5..94bf438c6 100644 --- a/pkg/services/control/server/set_shard_mode.go +++ b/pkg/services/control/server/set_shard_mode.go @@ -32,6 +32,8 @@ func (s *Server) SetShardMode(_ context.Context, req *control.SetShardModeReques m = mode.ReadOnly case control.ShardMode_DEGRADED: m = mode.Degraded + case control.ShardMode_DEGRADED_READ_ONLY: + m = mode.ReadOnly default: return nil, status.Error(codes.Internal, fmt.Sprintf("unknown shard mode: %s", requestedMode)) } diff --git a/pkg/services/control/service.pb.go b/pkg/services/control/service.pb.go index 4ad323c07..8bc8c282e 100644 Binary files a/pkg/services/control/service.pb.go and b/pkg/services/control/service.pb.go differ diff --git a/pkg/services/control/service_grpc.pb.go b/pkg/services/control/service_grpc.pb.go index bee17aee5..8632c4782 100644 Binary files a/pkg/services/control/service_grpc.pb.go and b/pkg/services/control/service_grpc.pb.go differ diff --git a/pkg/services/control/types.pb.go b/pkg/services/control/types.pb.go index c22eb0a7e..4c3d3c9ca 100644 Binary files a/pkg/services/control/types.pb.go and b/pkg/services/control/types.pb.go differ diff --git a/pkg/services/control/types.proto b/pkg/services/control/types.proto index f9c8cef2c..0729e4ebb 100644 --- a/pkg/services/control/types.proto +++ b/pkg/services/control/types.proto @@ -157,4 +157,7 @@ enum ShardMode { // Degraded. DEGRADED = 3; + + // DegradedReadOnly. + DEGRADED_READ_ONLY = 4; }