From 4944490ffbe2dc16b8d28d06395cf9478de42165 Mon Sep 17 00:00:00 2001
From: Evgenii Stratonikov <evgeniy@nspcc.ru>
Date: Mon, 18 Jul 2022 14:44:29 +0300
Subject: [PATCH] [#1559] local_object_storage: Move shard to the
 `DegradedReadOnly` mode

`Degraded` mode can be set by the administrator if needed.
Modifying operations in this mode can lead node into an inconsistent state
because metabase checks such as lock checking are not performed.

Signed-off-by: Evgenii Stratonikov <evgeniy@nspcc.ru>
---
 .../modules/morph/internal/types.pb.go        |  2 +-
 cmd/neofs-cli/modules/control/shards_list.go  |  2 ++
 .../modules/control/shards_set_mode.go        |  9 +++++---
 cmd/neofs-node/config/engine/shard/config.go  |  2 ++
 pkg/local_object_storage/engine/engine.go     |  2 +-
 pkg/local_object_storage/engine/error_test.go |  4 ++--
 pkg/local_object_storage/metabase/mode.go     |  8 +++----
 pkg/local_object_storage/shard/get.go         |  3 +--
 pkg/local_object_storage/shard/mode/mode.go   | 22 +++++++++++++------
 pkg/services/control/ir/service.pb.go         |  2 +-
 pkg/services/control/ir/service_grpc.pb.go    |  2 +-
 pkg/services/control/ir/types.pb.go           |  2 +-
 pkg/services/control/server/list_shards.go    |  2 ++
 pkg/services/control/server/set_shard_mode.go |  2 ++
 pkg/services/control/service.pb.go            |  2 +-
 pkg/services/control/service_grpc.pb.go       |  2 +-
 pkg/services/control/types.pb.go              | 19 ++++++++++------
 pkg/services/control/types.proto              |  3 +++
 18 files changed, 58 insertions(+), 32 deletions(-)

diff --git a/cmd/neofs-adm/internal/modules/morph/internal/types.pb.go b/cmd/neofs-adm/internal/modules/morph/internal/types.pb.go
index e76a29ae5..cdbe2045b 100644
--- a/cmd/neofs-adm/internal/modules/morph/internal/types.pb.go
+++ b/cmd/neofs-adm/internal/modules/morph/internal/types.pb.go
@@ -1,7 +1,7 @@
 // Code generated by protoc-gen-go. DO NOT EDIT.
 // versions:
 // 	protoc-gen-go v1.26.0
-// 	protoc        v3.19.4
+// 	protoc        v3.21.2
 // source: cmd/neofs-adm/internal/modules/morph/internal/types.proto
 
 package internal
diff --git a/cmd/neofs-cli/modules/control/shards_list.go b/cmd/neofs-cli/modules/control/shards_list.go
index 9ee986805..f6719d022 100644
--- a/cmd/neofs-cli/modules/control/shards_list.go
+++ b/cmd/neofs-cli/modules/control/shards_list.go
@@ -109,6 +109,8 @@ func shardModeToString(m control.ShardMode) string {
 		return "read-only"
 	case control.ShardMode_DEGRADED:
 		return "degraded"
+	case control.ShardMode_DEGRADED_READ_ONLY:
+		return "degraded-read-only"
 	default:
 		return "unknown"
 	}
diff --git a/cmd/neofs-cli/modules/control/shards_set_mode.go b/cmd/neofs-cli/modules/control/shards_set_mode.go
index 3c1eb6da9..625ec0434 100644
--- a/cmd/neofs-cli/modules/control/shards_set_mode.go
+++ b/cmd/neofs-cli/modules/control/shards_set_mode.go
@@ -17,9 +17,10 @@ const (
 	shardIDFlag          = "id"
 	shardClearErrorsFlag = "clear-errors"
 
-	shardModeReadOnly  = "read-only"
-	shardModeReadWrite = "read-write"
-	shardModeDegraded  = "degraded"
+	shardModeReadOnly         = "read-only"
+	shardModeReadWrite        = "read-write"
+	shardModeDegraded         = "degraded"
+	shardModeDegradedReadOnly = "degraded-read-only"
 )
 
 var setShardModeCmd = &cobra.Command{
@@ -60,6 +61,8 @@ func setShardMode(cmd *cobra.Command, _ []string) {
 		mode = control.ShardMode_READ_ONLY
 	case shardModeDegraded:
 		mode = control.ShardMode_DEGRADED
+	case shardModeDegradedReadOnly:
+		mode = control.ShardMode_DEGRADED_READ_ONLY
 	}
 
 	req := new(control.SetShardModeRequest)
diff --git a/cmd/neofs-node/config/engine/shard/config.go b/cmd/neofs-node/config/engine/shard/config.go
index 0bb8b2112..77c032a53 100644
--- a/cmd/neofs-node/config/engine/shard/config.go
+++ b/cmd/neofs-node/config/engine/shard/config.go
@@ -88,6 +88,8 @@ func (x *Config) Mode() (m mode.Mode) {
 		m = mode.ReadOnly
 	case "degraded":
 		m = mode.Degraded
+	case "degraded-read-only":
+		m = mode.DegradedReadOnly
 	default:
 		panic(fmt.Sprintf("unknown shard mode: %s", s))
 	}
diff --git a/pkg/local_object_storage/engine/engine.go b/pkg/local_object_storage/engine/engine.go
index 115cb7cd2..64041f623 100644
--- a/pkg/local_object_storage/engine/engine.go
+++ b/pkg/local_object_storage/engine/engine.go
@@ -51,7 +51,7 @@ func (e *StorageEngine) reportShardError(
 		return
 	}
 
-	err = sh.SetMode(mode.Degraded)
+	err = sh.SetMode(mode.DegradedReadOnly)
 	if err != nil {
 		e.log.Error("failed to move shard in degraded mode",
 			zap.Uint32("error count", errCount),
diff --git a/pkg/local_object_storage/engine/error_test.go b/pkg/local_object_storage/engine/error_test.go
index 9f049132d..c2907f3c5 100644
--- a/pkg/local_object_storage/engine/error_test.go
+++ b/pkg/local_object_storage/engine/error_test.go
@@ -125,7 +125,7 @@ func TestErrorReporting(t *testing.T) {
 		for i := uint32(0); i < 2; i++ {
 			_, err = e.Get(GetPrm{addr: object.AddressOf(obj)})
 			require.Error(t, err)
-			checkShardState(t, e, id[0], errThreshold+i, mode.Degraded)
+			checkShardState(t, e, id[0], errThreshold+i, mode.DegradedReadOnly)
 			checkShardState(t, e, id[1], 0, mode.ReadWrite)
 		}
 
@@ -193,7 +193,7 @@ func TestBlobstorFailback(t *testing.T) {
 		require.ErrorAs(t, err, &apistatus.ObjectOutOfRange{})
 	}
 
-	checkShardState(t, e, id[0], 1, mode.Degraded)
+	checkShardState(t, e, id[0], 1, mode.DegradedReadOnly)
 	checkShardState(t, e, id[1], 0, mode.ReadWrite)
 }
 
diff --git a/pkg/local_object_storage/metabase/mode.go b/pkg/local_object_storage/metabase/mode.go
index 9267bff3f..f5d4a8433 100644
--- a/pkg/local_object_storage/metabase/mode.go
+++ b/pkg/local_object_storage/metabase/mode.go
@@ -23,12 +23,12 @@ func (db *DB) SetMode(m mode.Mode) error {
 	}
 
 	var err error
-	switch m {
-	case mode.Degraded:
+	switch {
+	case m.NoMetabase():
 		db.boltDB = nil
-	case mode.ReadOnly:
+	case m.ReadOnly():
 		err = db.Open(true)
-	case mode.ReadWrite:
+	default:
 		err = db.Open(false)
 	}
 	if err == nil && !m.NoMetabase() && !m.ReadOnly() {
diff --git a/pkg/local_object_storage/shard/get.go b/pkg/local_object_storage/shard/get.go
index 16bd54c02..2647e74d9 100644
--- a/pkg/local_object_storage/shard/get.go
+++ b/pkg/local_object_storage/shard/get.go
@@ -6,7 +6,6 @@ import (
 	"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobovnicza"
 	"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor"
 	meta "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/metabase"
-	"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/shard/mode"
 	"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/writecache"
 	apistatus "github.com/nspcc-dev/neofs-sdk-go/client/status"
 	objectSDK "github.com/nspcc-dev/neofs-sdk-go/object"
@@ -123,7 +122,7 @@ func (s *Shard) fetchObjectData(addr oid.Address, skipMeta bool, big, small stor
 		mPrm.SetAddress(addr)
 
 		mRes, err := s.metaBase.Exists(mPrm)
-		if err != nil && s.GetMode() != mode.Degraded {
+		if err != nil && !s.GetMode().NoMetabase() {
 			return res, false, err
 		}
 		exists = mRes.Exists()
diff --git a/pkg/local_object_storage/shard/mode/mode.go b/pkg/local_object_storage/shard/mode/mode.go
index a0aef6e61..a22963612 100644
--- a/pkg/local_object_storage/shard/mode/mode.go
+++ b/pkg/local_object_storage/shard/mode/mode.go
@@ -3,19 +3,25 @@ package mode
 // Mode represents enumeration of Shard work modes.
 type Mode uint32
 
-// ReadWrite is a Mode value for shard that is available
-// for read and write operations. Default shard mode.
-const ReadWrite Mode = 0
+const (
+	// ReadWrite is a Mode value for shard that is available
+	// for read and write operations. Default shard mode.
+	ReadWrite Mode = 0
+
+	// DegradedReadOnly is a Mode value for shard that is set automatically
+	// after a certain number of errors is encountered. It is the same as
+	// `mode.ReadOnly` but also enables fallback algorithms for getting object
+	// in case metabase is corrupted.
+	DegradedReadOnly = Degraded | ReadOnly
+)
 
 const (
 	// ReadOnly is a Mode value for shard that does not
 	// accept write operation but is readable.
 	ReadOnly Mode = 1 << iota
 
-	// Degraded is a Mode value for shard that is set automatically
-	// after a certain number of errors is encountered. It is the same as
-	// `mode.ReadOnly` but also enables fallback algorithms for getting object
-	// in case metabase is corrupted.
+	// Degraded is a Mode value for shard when the metabase is unavailable.
+	// It is hard to perform some modifying operations in this mode, thus it can only be set by an administrator.
 	Degraded
 )
 
@@ -29,6 +35,8 @@ func (m Mode) String() string {
 		return "READ_ONLY"
 	case Degraded:
 		return "DEGRADED"
+	case DegradedReadOnly:
+		return "DEGRADED_READ_ONLY"
 	}
 }
 
diff --git a/pkg/services/control/ir/service.pb.go b/pkg/services/control/ir/service.pb.go
index bd6d26de9..61b58068e 100644
--- a/pkg/services/control/ir/service.pb.go
+++ b/pkg/services/control/ir/service.pb.go
@@ -1,7 +1,7 @@
 // Code generated by protoc-gen-go. DO NOT EDIT.
 // versions:
 // 	protoc-gen-go v1.26.0
-// 	protoc        v3.21.1
+// 	protoc        v3.21.2
 // source: pkg/services/control/ir/service.proto
 
 package control
diff --git a/pkg/services/control/ir/service_grpc.pb.go b/pkg/services/control/ir/service_grpc.pb.go
index edd848127..9ff723e63 100644
--- a/pkg/services/control/ir/service_grpc.pb.go
+++ b/pkg/services/control/ir/service_grpc.pb.go
@@ -1,7 +1,7 @@
 // Code generated by protoc-gen-go-grpc. DO NOT EDIT.
 // versions:
 // - protoc-gen-go-grpc v1.2.0
-// - protoc             v3.21.1
+// - protoc             v3.21.2
 // source: pkg/services/control/ir/service.proto
 
 package control
diff --git a/pkg/services/control/ir/types.pb.go b/pkg/services/control/ir/types.pb.go
index 13f4499ed..289f09cda 100644
--- a/pkg/services/control/ir/types.pb.go
+++ b/pkg/services/control/ir/types.pb.go
@@ -1,7 +1,7 @@
 // Code generated by protoc-gen-go. DO NOT EDIT.
 // versions:
 // 	protoc-gen-go v1.26.0
-// 	protoc        v3.21.1
+// 	protoc        v3.21.2
 // source: pkg/services/control/ir/types.proto
 
 package control
diff --git a/pkg/services/control/server/list_shards.go b/pkg/services/control/server/list_shards.go
index 8e6ac1b60..11a66dcd5 100644
--- a/pkg/services/control/server/list_shards.go
+++ b/pkg/services/control/server/list_shards.go
@@ -43,6 +43,8 @@ func (s *Server) ListShards(_ context.Context, req *control.ListShardsRequest) (
 			m = control.ShardMode_READ_ONLY
 		case mode.Degraded:
 			m = control.ShardMode_DEGRADED
+		case mode.DegradedReadOnly:
+			m = control.ShardMode_DEGRADED_READ_ONLY
 		default:
 			m = control.ShardMode_SHARD_MODE_UNDEFINED
 		}
diff --git a/pkg/services/control/server/set_shard_mode.go b/pkg/services/control/server/set_shard_mode.go
index 46ecdd6c5..94bf438c6 100644
--- a/pkg/services/control/server/set_shard_mode.go
+++ b/pkg/services/control/server/set_shard_mode.go
@@ -32,6 +32,8 @@ func (s *Server) SetShardMode(_ context.Context, req *control.SetShardModeReques
 		m = mode.ReadOnly
 	case control.ShardMode_DEGRADED:
 		m = mode.Degraded
+	case control.ShardMode_DEGRADED_READ_ONLY:
+		m = mode.ReadOnly
 	default:
 		return nil, status.Error(codes.Internal, fmt.Sprintf("unknown shard mode: %s", requestedMode))
 	}
diff --git a/pkg/services/control/service.pb.go b/pkg/services/control/service.pb.go
index 4ad323c07..8bc8c282e 100644
--- a/pkg/services/control/service.pb.go
+++ b/pkg/services/control/service.pb.go
@@ -1,7 +1,7 @@
 // Code generated by protoc-gen-go. DO NOT EDIT.
 // versions:
 // 	protoc-gen-go v1.26.0
-// 	protoc        v3.21.1
+// 	protoc        v3.21.2
 // source: pkg/services/control/service.proto
 
 package control
diff --git a/pkg/services/control/service_grpc.pb.go b/pkg/services/control/service_grpc.pb.go
index bee17aee5..8632c4782 100644
--- a/pkg/services/control/service_grpc.pb.go
+++ b/pkg/services/control/service_grpc.pb.go
@@ -1,7 +1,7 @@
 // Code generated by protoc-gen-go-grpc. DO NOT EDIT.
 // versions:
 // - protoc-gen-go-grpc v1.2.0
-// - protoc             v3.21.1
+// - protoc             v3.21.2
 // source: pkg/services/control/service.proto
 
 package control
diff --git a/pkg/services/control/types.pb.go b/pkg/services/control/types.pb.go
index c22eb0a7e..4c3d3c9ca 100644
--- a/pkg/services/control/types.pb.go
+++ b/pkg/services/control/types.pb.go
@@ -1,7 +1,7 @@
 // Code generated by protoc-gen-go. DO NOT EDIT.
 // versions:
 // 	protoc-gen-go v1.26.0
-// 	protoc        v3.21.1
+// 	protoc        v3.21.2
 // source: pkg/services/control/types.proto
 
 package control
@@ -146,6 +146,8 @@ const (
 	ShardMode_READ_ONLY ShardMode = 2
 	// Degraded.
 	ShardMode_DEGRADED ShardMode = 3
+	// DegradedReadOnly.
+	ShardMode_DEGRADED_READ_ONLY ShardMode = 4
 )
 
 // Enum value maps for ShardMode.
@@ -155,12 +157,14 @@ var (
 		1: "READ_WRITE",
 		2: "READ_ONLY",
 		3: "DEGRADED",
+		4: "DEGRADED_READ_ONLY",
 	}
 	ShardMode_value = map[string]int32{
 		"SHARD_MODE_UNDEFINED": 0,
 		"READ_WRITE":           1,
 		"READ_ONLY":            2,
 		"DEGRADED":             3,
+		"DEGRADED_READ_ONLY":   4,
 	}
 )
 
@@ -652,16 +656,17 @@ var file_pkg_services_control_types_proto_rawDesc = []byte{
 	0x4e, 0x45, 0x44, 0x10, 0x00, 0x12, 0x0c, 0x0a, 0x08, 0x53, 0x54, 0x41, 0x52, 0x54, 0x49, 0x4e,
 	0x47, 0x10, 0x01, 0x12, 0x09, 0x0a, 0x05, 0x52, 0x45, 0x41, 0x44, 0x59, 0x10, 0x02, 0x12, 0x11,
 	0x0a, 0x0d, 0x53, 0x48, 0x55, 0x54, 0x54, 0x49, 0x4e, 0x47, 0x5f, 0x44, 0x4f, 0x57, 0x4e, 0x10,
-	0x03, 0x2a, 0x52, 0x0a, 0x09, 0x53, 0x68, 0x61, 0x72, 0x64, 0x4d, 0x6f, 0x64, 0x65, 0x12, 0x18,
+	0x03, 0x2a, 0x6a, 0x0a, 0x09, 0x53, 0x68, 0x61, 0x72, 0x64, 0x4d, 0x6f, 0x64, 0x65, 0x12, 0x18,
 	0x0a, 0x14, 0x53, 0x48, 0x41, 0x52, 0x44, 0x5f, 0x4d, 0x4f, 0x44, 0x45, 0x5f, 0x55, 0x4e, 0x44,
 	0x45, 0x46, 0x49, 0x4e, 0x45, 0x44, 0x10, 0x00, 0x12, 0x0e, 0x0a, 0x0a, 0x52, 0x45, 0x41, 0x44,
 	0x5f, 0x57, 0x52, 0x49, 0x54, 0x45, 0x10, 0x01, 0x12, 0x0d, 0x0a, 0x09, 0x52, 0x45, 0x41, 0x44,
 	0x5f, 0x4f, 0x4e, 0x4c, 0x59, 0x10, 0x02, 0x12, 0x0c, 0x0a, 0x08, 0x44, 0x45, 0x47, 0x52, 0x41,
-	0x44, 0x45, 0x44, 0x10, 0x03, 0x42, 0x36, 0x5a, 0x34, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e,
-	0x63, 0x6f, 0x6d, 0x2f, 0x6e, 0x73, 0x70, 0x63, 0x63, 0x2d, 0x64, 0x65, 0x76, 0x2f, 0x6e, 0x65,
-	0x6f, 0x66, 0x73, 0x2d, 0x6e, 0x6f, 0x64, 0x65, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x73, 0x65, 0x72,
-	0x76, 0x69, 0x63, 0x65, 0x73, 0x2f, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x62, 0x06, 0x70,
-	0x72, 0x6f, 0x74, 0x6f, 0x33,
+	0x44, 0x45, 0x44, 0x10, 0x03, 0x12, 0x16, 0x0a, 0x12, 0x44, 0x45, 0x47, 0x52, 0x41, 0x44, 0x45,
+	0x44, 0x5f, 0x52, 0x45, 0x41, 0x44, 0x5f, 0x4f, 0x4e, 0x4c, 0x59, 0x10, 0x04, 0x42, 0x36, 0x5a,
+	0x34, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x6e, 0x73, 0x70, 0x63,
+	0x63, 0x2d, 0x64, 0x65, 0x76, 0x2f, 0x6e, 0x65, 0x6f, 0x66, 0x73, 0x2d, 0x6e, 0x6f, 0x64, 0x65,
+	0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2f, 0x63, 0x6f,
+	0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
 }
 
 var (
diff --git a/pkg/services/control/types.proto b/pkg/services/control/types.proto
index f9c8cef2c..0729e4ebb 100644
--- a/pkg/services/control/types.proto
+++ b/pkg/services/control/types.proto
@@ -157,4 +157,7 @@ enum ShardMode {
 
     // Degraded.
     DEGRADED = 3;
+
+    // DegradedReadOnly.
+    DEGRADED_READ_ONLY = 4;
 }