[#1559] local_object_storage: Move shard to the DegradedReadOnly mode

`Degraded` mode can be set by the administrator if needed.
Modifying operations in this mode can lead node into an inconsistent state
because metabase checks such as lock checking are not performed.

Signed-off-by: Evgenii Stratonikov <evgeniy@nspcc.ru>

cmd/neofs-adm/internal/modules/morph/internal/types.pb.go

@@ -1,7 +1,7 @@
 // Code generated by protoc-gen-go. DO NOT EDIT.
 // versions:
 // 	protoc-gen-go v1.26.0
-// 	protoc        v3.19.4
+// 	protoc        v3.21.2
 // source: cmd/neofs-adm/internal/modules/morph/internal/types.proto
 
 package internal

cmd/neofs-cli/modules/control/shards_list.go

@@ -109,6 +109,8 @@ func shardModeToString(m control.ShardMode) string {
 		return "read-only"
 	case control.ShardMode_DEGRADED:
 		return "degraded"
+	case control.ShardMode_DEGRADED_READ_ONLY:
+		return "degraded-read-only"
 	default:
 		return "unknown"
 	}

cmd/neofs-cli/modules/control/shards_set_mode.go

@@ -20,6 +20,7 @@ const (
 	shardModeReadOnly         = "read-only"
 	shardModeReadWrite        = "read-write"
 	shardModeDegraded         = "degraded"
+	shardModeDegradedReadOnly = "degraded-read-only"
 )
 
 var setShardModeCmd = &cobra.Command{
@@ -60,6 +61,8 @@ func setShardMode(cmd *cobra.Command, _ []string) {
 		mode = control.ShardMode_READ_ONLY
 	case shardModeDegraded:
 		mode = control.ShardMode_DEGRADED
+	case shardModeDegradedReadOnly:
+		mode = control.ShardMode_DEGRADED_READ_ONLY
 	}
 
 	req := new(control.SetShardModeRequest)

cmd/neofs-node/config/engine/shard/config.go

@@ -88,6 +88,8 @@ func (x *Config) Mode() (m mode.Mode) {
 		m = mode.ReadOnly
 	case "degraded":
 		m = mode.Degraded
+	case "degraded-read-only":
+		m = mode.DegradedReadOnly
 	default:
 		panic(fmt.Sprintf("unknown shard mode: %s", s))
 	}

pkg/local_object_storage/engine/engine.go

@@ -51,7 +51,7 @@ func (e *StorageEngine) reportShardError(
 		return
 	}
 
-	err = sh.SetMode(mode.Degraded)
+	err = sh.SetMode(mode.DegradedReadOnly)
 	if err != nil {
 		e.log.Error("failed to move shard in degraded mode",
 			zap.Uint32("error count", errCount),

pkg/local_object_storage/engine/error_test.go

@@ -125,7 +125,7 @@ func TestErrorReporting(t *testing.T) {
 		for i := uint32(0); i < 2; i++ {
 			_, err = e.Get(GetPrm{addr: object.AddressOf(obj)})
 			require.Error(t, err)
-			checkShardState(t, e, id[0], errThreshold+i, mode.Degraded)
+			checkShardState(t, e, id[0], errThreshold+i, mode.DegradedReadOnly)
 			checkShardState(t, e, id[1], 0, mode.ReadWrite)
 		}
 
@@ -193,7 +193,7 @@ func TestBlobstorFailback(t *testing.T) {
 		require.ErrorAs(t, err, &apistatus.ObjectOutOfRange{})
 	}
 
-	checkShardState(t, e, id[0], 1, mode.Degraded)
+	checkShardState(t, e, id[0], 1, mode.DegradedReadOnly)
 	checkShardState(t, e, id[1], 0, mode.ReadWrite)
 }
 

pkg/local_object_storage/metabase/mode.go

@@ -23,12 +23,12 @@ func (db *DB) SetMode(m mode.Mode) error {
 	}
 
 	var err error
-	switch m {
+	switch {
-	case mode.Degraded:
+	case m.NoMetabase():
 		db.boltDB = nil
-	case mode.ReadOnly:
+	case m.ReadOnly():
 		err = db.Open(true)
-	case mode.ReadWrite:
+	default:
 		err = db.Open(false)
 	}
 	if err == nil && !m.NoMetabase() && !m.ReadOnly() {

pkg/local_object_storage/shard/get.go

@@ -6,7 +6,6 @@ import (
 	"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobovnicza"
 	"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor"
 	meta "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/metabase"
-	"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/shard/mode"
 	"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/writecache"
 	apistatus "github.com/nspcc-dev/neofs-sdk-go/client/status"
 	objectSDK "github.com/nspcc-dev/neofs-sdk-go/object"
@@ -123,7 +122,7 @@ func (s *Shard) fetchObjectData(addr oid.Address, skipMeta bool, big, small stor
 		mPrm.SetAddress(addr)
 
 		mRes, err := s.metaBase.Exists(mPrm)
-		if err != nil && s.GetMode() != mode.Degraded {
+		if err != nil && !s.GetMode().NoMetabase() {
 			return res, false, err
 		}
 		exists = mRes.Exists()

pkg/local_object_storage/shard/mode/mode.go

@@ -3,19 +3,25 @@ package mode
 // Mode represents enumeration of Shard work modes.
 type Mode uint32
 
+const (
 	// ReadWrite is a Mode value for shard that is available
 	// for read and write operations. Default shard mode.
-const ReadWrite Mode = 0
+	ReadWrite Mode = 0
+
+	// DegradedReadOnly is a Mode value for shard that is set automatically
+	// after a certain number of errors is encountered. It is the same as
+	// `mode.ReadOnly` but also enables fallback algorithms for getting object
+	// in case metabase is corrupted.
+	DegradedReadOnly = Degraded | ReadOnly
+)
 
 const (
 	// ReadOnly is a Mode value for shard that does not
 	// accept write operation but is readable.
 	ReadOnly Mode = 1 << iota
 
-	// Degraded is a Mode value for shard that is set automatically
+	// Degraded is a Mode value for shard when the metabase is unavailable.
-	// after a certain number of errors is encountered. It is the same as
+	// It is hard to perform some modifying operations in this mode, thus it can only be set by an administrator.
-	// `mode.ReadOnly` but also enables fallback algorithms for getting object
-	// in case metabase is corrupted.
 	Degraded
 )
 
@@ -29,6 +35,8 @@ func (m Mode) String() string {
 		return "READ_ONLY"
 	case Degraded:
 		return "DEGRADED"
+	case DegradedReadOnly:
+		return "DEGRADED_READ_ONLY"
 	}
 }
 

pkg/services/control/ir/service.pb.go

@@ -1,7 +1,7 @@
 // Code generated by protoc-gen-go. DO NOT EDIT.
 // versions:
 // 	protoc-gen-go v1.26.0
-// 	protoc        v3.21.1
+// 	protoc        v3.21.2
 // source: pkg/services/control/ir/service.proto
 
 package control

pkg/services/control/ir/service_grpc.pb.go

@@ -1,7 +1,7 @@
 // Code generated by protoc-gen-go-grpc. DO NOT EDIT.
 // versions:
 // - protoc-gen-go-grpc v1.2.0
-// - protoc             v3.21.1
+// - protoc             v3.21.2
 // source: pkg/services/control/ir/service.proto
 
 package control

pkg/services/control/ir/types.pb.go

@@ -1,7 +1,7 @@
 // Code generated by protoc-gen-go. DO NOT EDIT.
 // versions:
 // 	protoc-gen-go v1.26.0
-// 	protoc        v3.21.1
+// 	protoc        v3.21.2
 // source: pkg/services/control/ir/types.proto
 
 package control

pkg/services/control/server/list_shards.go

@@ -43,6 +43,8 @@ func (s *Server) ListShards(_ context.Context, req *control.ListShardsRequest) (
 			m = control.ShardMode_READ_ONLY
 		case mode.Degraded:
 			m = control.ShardMode_DEGRADED
+		case mode.DegradedReadOnly:
+			m = control.ShardMode_DEGRADED_READ_ONLY
 		default:
 			m = control.ShardMode_SHARD_MODE_UNDEFINED
 		}

pkg/services/control/server/set_shard_mode.go

@@ -32,6 +32,8 @@ func (s *Server) SetShardMode(_ context.Context, req *control.SetShardModeReques
 		m = mode.ReadOnly
 	case control.ShardMode_DEGRADED:
 		m = mode.Degraded
+	case control.ShardMode_DEGRADED_READ_ONLY:
+		m = mode.ReadOnly
 	default:
 		return nil, status.Error(codes.Internal, fmt.Sprintf("unknown shard mode: %s", requestedMode))
 	}

pkg/services/control/service.pb.go

@@ -1,7 +1,7 @@
 // Code generated by protoc-gen-go. DO NOT EDIT.
 // versions:
 // 	protoc-gen-go v1.26.0
-// 	protoc        v3.21.1
+// 	protoc        v3.21.2
 // source: pkg/services/control/service.proto
 
 package control

pkg/services/control/service_grpc.pb.go

@@ -1,7 +1,7 @@
 // Code generated by protoc-gen-go-grpc. DO NOT EDIT.
 // versions:
 // - protoc-gen-go-grpc v1.2.0
-// - protoc             v3.21.1
+// - protoc             v3.21.2
 // source: pkg/services/control/service.proto
 
 package control

pkg/services/control/types.pb.go

@@ -1,7 +1,7 @@
 // Code generated by protoc-gen-go. DO NOT EDIT.
 // versions:
 // 	protoc-gen-go v1.26.0
-// 	protoc        v3.21.1
+// 	protoc        v3.21.2
 // source: pkg/services/control/types.proto
 
 package control
@@ -146,6 +146,8 @@ const (
 	ShardMode_READ_ONLY ShardMode = 2
 	// Degraded.
 	ShardMode_DEGRADED ShardMode = 3
+	// DegradedReadOnly.
+	ShardMode_DEGRADED_READ_ONLY ShardMode = 4
 )
 
 // Enum value maps for ShardMode.
@@ -155,12 +157,14 @@ var (
 		1: "READ_WRITE",
 		2: "READ_ONLY",
 		3: "DEGRADED",
+		4: "DEGRADED_READ_ONLY",
 	}
 	ShardMode_value = map[string]int32{
 		"SHARD_MODE_UNDEFINED": 0,
 		"READ_WRITE":           1,
 		"READ_ONLY":            2,
 		"DEGRADED":             3,
+		"DEGRADED_READ_ONLY":   4,
 	}
 )
 
@@ -652,16 +656,17 @@ var file_pkg_services_control_types_proto_rawDesc = []byte{
 	0x4e, 0x45, 0x44, 0x10, 0x00, 0x12, 0x0c, 0x0a, 0x08, 0x53, 0x54, 0x41, 0x52, 0x54, 0x49, 0x4e,
 	0x47, 0x10, 0x01, 0x12, 0x09, 0x0a, 0x05, 0x52, 0x45, 0x41, 0x44, 0x59, 0x10, 0x02, 0x12, 0x11,
 	0x0a, 0x0d, 0x53, 0x48, 0x55, 0x54, 0x54, 0x49, 0x4e, 0x47, 0x5f, 0x44, 0x4f, 0x57, 0x4e, 0x10,
-	0x03, 0x2a, 0x52, 0x0a, 0x09, 0x53, 0x68, 0x61, 0x72, 0x64, 0x4d, 0x6f, 0x64, 0x65, 0x12, 0x18,
+	0x03, 0x2a, 0x6a, 0x0a, 0x09, 0x53, 0x68, 0x61, 0x72, 0x64, 0x4d, 0x6f, 0x64, 0x65, 0x12, 0x18,
 	0x0a, 0x14, 0x53, 0x48, 0x41, 0x52, 0x44, 0x5f, 0x4d, 0x4f, 0x44, 0x45, 0x5f, 0x55, 0x4e, 0x44,
 	0x45, 0x46, 0x49, 0x4e, 0x45, 0x44, 0x10, 0x00, 0x12, 0x0e, 0x0a, 0x0a, 0x52, 0x45, 0x41, 0x44,
 	0x5f, 0x57, 0x52, 0x49, 0x54, 0x45, 0x10, 0x01, 0x12, 0x0d, 0x0a, 0x09, 0x52, 0x45, 0x41, 0x44,
 	0x5f, 0x4f, 0x4e, 0x4c, 0x59, 0x10, 0x02, 0x12, 0x0c, 0x0a, 0x08, 0x44, 0x45, 0x47, 0x52, 0x41,
-	0x44, 0x45, 0x44, 0x10, 0x03, 0x42, 0x36, 0x5a, 0x34, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e,
+	0x44, 0x45, 0x44, 0x10, 0x03, 0x12, 0x16, 0x0a, 0x12, 0x44, 0x45, 0x47, 0x52, 0x41, 0x44, 0x45,
-	0x63, 0x6f, 0x6d, 0x2f, 0x6e, 0x73, 0x70, 0x63, 0x63, 0x2d, 0x64, 0x65, 0x76, 0x2f, 0x6e, 0x65,
+	0x44, 0x5f, 0x52, 0x45, 0x41, 0x44, 0x5f, 0x4f, 0x4e, 0x4c, 0x59, 0x10, 0x04, 0x42, 0x36, 0x5a,
-	0x6f, 0x66, 0x73, 0x2d, 0x6e, 0x6f, 0x64, 0x65, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x73, 0x65, 0x72,
+	0x34, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x6e, 0x73, 0x70, 0x63,
-	0x76, 0x69, 0x63, 0x65, 0x73, 0x2f, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x62, 0x06, 0x70,
+	0x63, 0x2d, 0x64, 0x65, 0x76, 0x2f, 0x6e, 0x65, 0x6f, 0x66, 0x73, 0x2d, 0x6e, 0x6f, 0x64, 0x65,
-	0x72, 0x6f, 0x74, 0x6f, 0x33,
+	0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2f, 0x63, 0x6f,
+	0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
 }
 
 var (

pkg/services/control/types.proto

@@ -157,4 +157,7 @@ enum ShardMode {
 
     // Degraded.
     DEGRADED = 3;
+
+    // DegradedReadOnly.
+    DEGRADED_READ_ONLY = 4;
 }