From 4944490ffbe2dc16b8d28d06395cf9478de42165 Mon Sep 17 00:00:00 2001
From: Evgenii Stratonikov <evgeniy@nspcc.ru>
Date: Mon, 18 Jul 2022 14:44:29 +0300
Subject: [PATCH] [#1559] local_object_storage: Move shard to the
 `DegradedReadOnly` mode

`Degraded` mode can be set by the administrator if needed.
Modifying operations in this mode can lead node into an inconsistent state
because metabase checks such as lock checking are not performed.

Signed-off-by: Evgenii Stratonikov <evgeniy@nspcc.ru>
---
 .../modules/morph/internal/types.pb.go        |  2 +-
 cmd/neofs-cli/modules/control/shards_list.go  |  2 ++
 .../modules/control/shards_set_mode.go        |  9 +++++---
 cmd/neofs-node/config/engine/shard/config.go  |  2 ++
 pkg/local_object_storage/engine/engine.go     |  2 +-
 pkg/local_object_storage/engine/error_test.go |  4 ++--
 pkg/local_object_storage/metabase/mode.go     |  8 +++----
 pkg/local_object_storage/shard/get.go         |  3 +--
 pkg/local_object_storage/shard/mode/mode.go   | 22 +++++++++++++------
 pkg/services/control/ir/service.pb.go         |  2 +-
 pkg/services/control/ir/service_grpc.pb.go    |  2 +-
 pkg/services/control/ir/types.pb.go           |  2 +-
 pkg/services/control/server/list_shards.go    |  2 ++
 pkg/services/control/server/set_shard_mode.go |  2 ++
 pkg/services/control/service.pb.go            |  2 +-
 pkg/services/control/service_grpc.pb.go       |  2 +-
 pkg/services/control/types.pb.go              | 19 ++++++++++------
 pkg/services/control/types.proto              |  3 +++
 18 files changed, 58 insertions(+), 32 deletions(-)

diff --git a/cmd/neofs-adm/internal/modules/morph/internal/types.pb.go b/cmd/neofs-adm/internal/modules/morph/internal/types.pb.go
index e76a29ae..cdbe2045 100644
--- a/cmd/neofs-adm/internal/modules/morph/internal/types.pb.go
+++ b/cmd/neofs-adm/internal/modules/morph/internal/types.pb.go
@@ -1,7 +1,7 @@
 // Code generated by protoc-gen-go. DO NOT EDIT.
 // versions:
 // 	protoc-gen-go v1.26.0
-// 	protoc        v3.19.4
+// 	protoc        v3.21.2
 // source: cmd/neofs-adm/internal/modules/morph/internal/types.proto
 
 package internal
diff --git a/cmd/neofs-cli/modules/control/shards_list.go b/cmd/neofs-cli/modules/control/shards_list.go
index 9ee98680..f6719d02 100644
--- a/cmd/neofs-cli/modules/control/shards_list.go
+++ b/cmd/neofs-cli/modules/control/shards_list.go
@@ -109,6 +109,8 @@ func shardModeToString(m control.ShardMode) string {
 		return "read-only"
 	case control.ShardMode_DEGRADED:
 		return "degraded"
+	case control.ShardMode_DEGRADED_READ_ONLY:
+		return "degraded-read-only"
 	default:
 		return "unknown"
 	}
diff --git a/cmd/neofs-cli/modules/control/shards_set_mode.go b/cmd/neofs-cli/modules/control/shards_set_mode.go
index 3c1eb6da..625ec043 100644
--- a/cmd/neofs-cli/modules/control/shards_set_mode.go
+++ b/cmd/neofs-cli/modules/control/shards_set_mode.go
@@ -17,9 +17,10 @@ const (
 	shardIDFlag          = "id"
 	shardClearErrorsFlag = "clear-errors"
 
-	shardModeReadOnly  = "read-only"
-	shardModeReadWrite = "read-write"
-	shardModeDegraded  = "degraded"
+	shardModeReadOnly         = "read-only"
+	shardModeReadWrite        = "read-write"
+	shardModeDegraded         = "degraded"
+	shardModeDegradedReadOnly = "degraded-read-only"
 )
 
 var setShardModeCmd = &cobra.Command{
@@ -60,6 +61,8 @@ func setShardMode(cmd *cobra.Command, _ []string) {
 		mode = control.ShardMode_READ_ONLY
 	case shardModeDegraded:
 		mode = control.ShardMode_DEGRADED
+	case shardModeDegradedReadOnly:
+		mode = control.ShardMode_DEGRADED_READ_ONLY
 	}
 
 	req := new(control.SetShardModeRequest)
diff --git a/cmd/neofs-node/config/engine/shard/config.go b/cmd/neofs-node/config/engine/shard/config.go
index 0bb8b211..77c032a5 100644
--- a/cmd/neofs-node/config/engine/shard/config.go
+++ b/cmd/neofs-node/config/engine/shard/config.go
@@ -88,6 +88,8 @@ func (x *Config) Mode() (m mode.Mode) {
 		m = mode.ReadOnly
 	case "degraded":
 		m = mode.Degraded
+	case "degraded-read-only":
+		m = mode.DegradedReadOnly
 	default:
 		panic(fmt.Sprintf("unknown shard mode: %s", s))
 	}
diff --git a/pkg/local_object_storage/engine/engine.go b/pkg/local_object_storage/engine/engine.go
index 115cb7cd..64041f62 100644
--- a/pkg/local_object_storage/engine/engine.go
+++ b/pkg/local_object_storage/engine/engine.go
@@ -51,7 +51,7 @@ func (e *StorageEngine) reportShardError(
 		return
 	}
 
-	err = sh.SetMode(mode.Degraded)
+	err = sh.SetMode(mode.DegradedReadOnly)
 	if err != nil {
 		e.log.Error("failed to move shard in degraded mode",
 			zap.Uint32("error count", errCount),
diff --git a/pkg/local_object_storage/engine/error_test.go b/pkg/local_object_storage/engine/error_test.go
index 9f049132..c2907f3c 100644
--- a/pkg/local_object_storage/engine/error_test.go
+++ b/pkg/local_object_storage/engine/error_test.go
@@ -125,7 +125,7 @@ func TestErrorReporting(t *testing.T) {
 		for i := uint32(0); i < 2; i++ {
 			_, err = e.Get(GetPrm{addr: object.AddressOf(obj)})
 			require.Error(t, err)
-			checkShardState(t, e, id[0], errThreshold+i, mode.Degraded)
+			checkShardState(t, e, id[0], errThreshold+i, mode.DegradedReadOnly)
 			checkShardState(t, e, id[1], 0, mode.ReadWrite)
 		}
 
@@ -193,7 +193,7 @@ func TestBlobstorFailback(t *testing.T) {
 		require.ErrorAs(t, err, &apistatus.ObjectOutOfRange{})
 	}
 
-	checkShardState(t, e, id[0], 1, mode.Degraded)
+	checkShardState(t, e, id[0], 1, mode.DegradedReadOnly)
 	checkShardState(t, e, id[1], 0, mode.ReadWrite)
 }
 
diff --git a/pkg/local_object_storage/metabase/mode.go b/pkg/local_object_storage/metabase/mode.go
index 9267bff3..f5d4a843 100644
--- a/pkg/local_object_storage/metabase/mode.go
+++ b/pkg/local_object_storage/metabase/mode.go
@@ -23,12 +23,12 @@ func (db *DB) SetMode(m mode.Mode) error {
 	}
 
 	var err error
-	switch m {
-	case mode.Degraded:
+	switch {
+	case m.NoMetabase():
 		db.boltDB = nil
-	case mode.ReadOnly:
+	case m.ReadOnly():
 		err = db.Open(true)
-	case mode.ReadWrite:
+	default:
 		err = db.Open(false)
 	}
 	if err == nil && !m.NoMetabase() && !m.ReadOnly() {
diff --git a/pkg/local_object_storage/shard/get.go b/pkg/local_object_storage/shard/get.go
index 16bd54c0..2647e74d 100644
--- a/pkg/local_object_storage/shard/get.go
+++ b/pkg/local_object_storage/shard/get.go
@@ -6,7 +6,6 @@ import (
 	"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobovnicza"
 	"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor"
 	meta "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/metabase"
-	"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/shard/mode"
 	"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/writecache"
 	apistatus "github.com/nspcc-dev/neofs-sdk-go/client/status"
 	objectSDK "github.com/nspcc-dev/neofs-sdk-go/object"
@@ -123,7 +122,7 @@ func (s *Shard) fetchObjectData(addr oid.Address, skipMeta bool, big, small stor
 		mPrm.SetAddress(addr)
 
 		mRes, err := s.metaBase.Exists(mPrm)
-		if err != nil && s.GetMode() != mode.Degraded {
+		if err != nil && !s.GetMode().NoMetabase() {
 			return res, false, err
 		}
 		exists = mRes.Exists()
diff --git a/pkg/local_object_storage/shard/mode/mode.go b/pkg/local_object_storage/shard/mode/mode.go
index a0aef6e6..a2296361 100644
--- a/pkg/local_object_storage/shard/mode/mode.go
+++ b/pkg/local_object_storage/shard/mode/mode.go
@@ -3,19 +3,25 @@ package mode
 // Mode represents enumeration of Shard work modes.
 type Mode uint32
 
-// ReadWrite is a Mode value for shard that is available
-// for read and write operations. Default shard mode.
-const ReadWrite Mode = 0
+const (
+	// ReadWrite is a Mode value for shard that is available
+	// for read and write operations. Default shard mode.
+	ReadWrite Mode = 0
+
+	// DegradedReadOnly is a Mode value for shard that is set automatically
+	// after a certain number of errors is encountered. It is the same as
+	// `mode.ReadOnly` but also enables fallback algorithms for getting object
+	// in case metabase is corrupted.
+	DegradedReadOnly = Degraded | ReadOnly
+)
 
 const (
 	// ReadOnly is a Mode value for shard that does not
 	// accept write operation but is readable.
 	ReadOnly Mode = 1 << iota
 
-	// Degraded is a Mode value for shard that is set automatically
-	// after a certain number of errors is encountered. It is the same as
-	// `mode.ReadOnly` but also enables fallback algorithms for getting object
-	// in case metabase is corrupted.
+	// Degraded is a Mode value for shard when the metabase is unavailable.
+	// It is hard to perform some modifying operations in this mode, thus it can only be set by an administrator.
 	Degraded
 )
 
@@ -29,6 +35,8 @@ func (m Mode) String() string {
 		return "READ_ONLY"
 	case Degraded:
 		return "DEGRADED"
+	case DegradedReadOnly:
+		return "DEGRADED_READ_ONLY"
 	}
 }
 
diff --git a/pkg/services/control/ir/service.pb.go b/pkg/services/control/ir/service.pb.go
index bd6d26de..61b58068 100644
--- a/pkg/services/control/ir/service.pb.go
+++ b/pkg/services/control/ir/service.pb.go
@@ -1,7 +1,7 @@
 // Code generated by protoc-gen-go. DO NOT EDIT.
 // versions:
 // 	protoc-gen-go v1.26.0
-// 	protoc        v3.21.1
+// 	protoc        v3.21.2
 // source: pkg/services/control/ir/service.proto
 
 package control
diff --git a/pkg/services/control/ir/service_grpc.pb.go b/pkg/services/control/ir/service_grpc.pb.go
index edd84812..9ff723e6 100644
--- a/pkg/services/control/ir/service_grpc.pb.go
+++ b/pkg/services/control/ir/service_grpc.pb.go
@@ -1,7 +1,7 @@
 // Code generated by protoc-gen-go-grpc. DO NOT EDIT.
 // versions:
 // - protoc-gen-go-grpc v1.2.0
-// - protoc             v3.21.1
+// - protoc             v3.21.2
 // source: pkg/services/control/ir/service.proto
 
 package control
diff --git a/pkg/services/control/ir/types.pb.go b/pkg/services/control/ir/types.pb.go
index 13f4499e..289f09cd 100644
--- a/pkg/services/control/ir/types.pb.go
+++ b/pkg/services/control/ir/types.pb.go
@@ -1,7 +1,7 @@
 // Code generated by protoc-gen-go. DO NOT EDIT.
 // versions:
 // 	protoc-gen-go v1.26.0
-// 	protoc        v3.21.1
+// 	protoc        v3.21.2
 // source: pkg/services/control/ir/types.proto
 
 package control
diff --git a/pkg/services/control/server/list_shards.go b/pkg/services/control/server/list_shards.go
index 8e6ac1b6..11a66dcd 100644
--- a/pkg/services/control/server/list_shards.go
+++ b/pkg/services/control/server/list_shards.go
@@ -43,6 +43,8 @@ func (s *Server) ListShards(_ context.Context, req *control.ListShardsRequest) (
 			m = control.ShardMode_READ_ONLY
 		case mode.Degraded:
 			m = control.ShardMode_DEGRADED
+		case mode.DegradedReadOnly:
+			m = control.ShardMode_DEGRADED_READ_ONLY
 		default:
 			m = control.ShardMode_SHARD_MODE_UNDEFINED
 		}
diff --git a/pkg/services/control/server/set_shard_mode.go b/pkg/services/control/server/set_shard_mode.go
index 46ecdd6c..94bf438c 100644
--- a/pkg/services/control/server/set_shard_mode.go
+++ b/pkg/services/control/server/set_shard_mode.go
@@ -32,6 +32,8 @@ func (s *Server) SetShardMode(_ context.Context, req *control.SetShardModeReques
 		m = mode.ReadOnly
 	case control.ShardMode_DEGRADED:
 		m = mode.Degraded
+	case control.ShardMode_DEGRADED_READ_ONLY:
+		m = mode.ReadOnly
 	default:
 		return nil, status.Error(codes.Internal, fmt.Sprintf("unknown shard mode: %s", requestedMode))
 	}
diff --git a/pkg/services/control/service.pb.go b/pkg/services/control/service.pb.go
index 4ad323c0..8bc8c282 100644
--- a/pkg/services/control/service.pb.go
+++ b/pkg/services/control/service.pb.go
@@ -1,7 +1,7 @@
 // Code generated by protoc-gen-go. DO NOT EDIT.
 // versions:
 // 	protoc-gen-go v1.26.0
-// 	protoc        v3.21.1
+// 	protoc        v3.21.2
 // source: pkg/services/control/service.proto
 
 package control
diff --git a/pkg/services/control/service_grpc.pb.go b/pkg/services/control/service_grpc.pb.go
index bee17aee..8632c478 100644
--- a/pkg/services/control/service_grpc.pb.go
+++ b/pkg/services/control/service_grpc.pb.go
@@ -1,7 +1,7 @@
 // Code generated by protoc-gen-go-grpc. DO NOT EDIT.
 // versions:
 // - protoc-gen-go-grpc v1.2.0
-// - protoc             v3.21.1
+// - protoc             v3.21.2
 // source: pkg/services/control/service.proto
 
 package control
diff --git a/pkg/services/control/types.pb.go b/pkg/services/control/types.pb.go
index c22eb0a7..4c3d3c9c 100644
--- a/pkg/services/control/types.pb.go
+++ b/pkg/services/control/types.pb.go
@@ -1,7 +1,7 @@
 // Code generated by protoc-gen-go. DO NOT EDIT.
 // versions:
 // 	protoc-gen-go v1.26.0
-// 	protoc        v3.21.1
+// 	protoc        v3.21.2
 // source: pkg/services/control/types.proto
 
 package control
@@ -146,6 +146,8 @@ const (
 	ShardMode_READ_ONLY ShardMode = 2
 	// Degraded.
 	ShardMode_DEGRADED ShardMode = 3
+	// DegradedReadOnly.
+	ShardMode_DEGRADED_READ_ONLY ShardMode = 4
 )
 
 // Enum value maps for ShardMode.
@@ -155,12 +157,14 @@ var (
 		1: "READ_WRITE",
 		2: "READ_ONLY",
 		3: "DEGRADED",
+		4: "DEGRADED_READ_ONLY",
 	}
 	ShardMode_value = map[string]int32{
 		"SHARD_MODE_UNDEFINED": 0,
 		"READ_WRITE":           1,
 		"READ_ONLY":            2,
 		"DEGRADED":             3,
+		"DEGRADED_READ_ONLY":   4,
 	}
 )
 
@@ -652,16 +656,17 @@ var file_pkg_services_control_types_proto_rawDesc = []byte{
 	0x4e, 0x45, 0x44, 0x10, 0x00, 0x12, 0x0c, 0x0a, 0x08, 0x53, 0x54, 0x41, 0x52, 0x54, 0x49, 0x4e,
 	0x47, 0x10, 0x01, 0x12, 0x09, 0x0a, 0x05, 0x52, 0x45, 0x41, 0x44, 0x59, 0x10, 0x02, 0x12, 0x11,
 	0x0a, 0x0d, 0x53, 0x48, 0x55, 0x54, 0x54, 0x49, 0x4e, 0x47, 0x5f, 0x44, 0x4f, 0x57, 0x4e, 0x10,
-	0x03, 0x2a, 0x52, 0x0a, 0x09, 0x53, 0x68, 0x61, 0x72, 0x64, 0x4d, 0x6f, 0x64, 0x65, 0x12, 0x18,
+	0x03, 0x2a, 0x6a, 0x0a, 0x09, 0x53, 0x68, 0x61, 0x72, 0x64, 0x4d, 0x6f, 0x64, 0x65, 0x12, 0x18,
 	0x0a, 0x14, 0x53, 0x48, 0x41, 0x52, 0x44, 0x5f, 0x4d, 0x4f, 0x44, 0x45, 0x5f, 0x55, 0x4e, 0x44,
 	0x45, 0x46, 0x49, 0x4e, 0x45, 0x44, 0x10, 0x00, 0x12, 0x0e, 0x0a, 0x0a, 0x52, 0x45, 0x41, 0x44,
 	0x5f, 0x57, 0x52, 0x49, 0x54, 0x45, 0x10, 0x01, 0x12, 0x0d, 0x0a, 0x09, 0x52, 0x45, 0x41, 0x44,
 	0x5f, 0x4f, 0x4e, 0x4c, 0x59, 0x10, 0x02, 0x12, 0x0c, 0x0a, 0x08, 0x44, 0x45, 0x47, 0x52, 0x41,
-	0x44, 0x45, 0x44, 0x10, 0x03, 0x42, 0x36, 0x5a, 0x34, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e,
-	0x63, 0x6f, 0x6d, 0x2f, 0x6e, 0x73, 0x70, 0x63, 0x63, 0x2d, 0x64, 0x65, 0x76, 0x2f, 0x6e, 0x65,
-	0x6f, 0x66, 0x73, 0x2d, 0x6e, 0x6f, 0x64, 0x65, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x73, 0x65, 0x72,
-	0x76, 0x69, 0x63, 0x65, 0x73, 0x2f, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x62, 0x06, 0x70,
-	0x72, 0x6f, 0x74, 0x6f, 0x33,
+	0x44, 0x45, 0x44, 0x10, 0x03, 0x12, 0x16, 0x0a, 0x12, 0x44, 0x45, 0x47, 0x52, 0x41, 0x44, 0x45,
+	0x44, 0x5f, 0x52, 0x45, 0x41, 0x44, 0x5f, 0x4f, 0x4e, 0x4c, 0x59, 0x10, 0x04, 0x42, 0x36, 0x5a,
+	0x34, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x6e, 0x73, 0x70, 0x63,
+	0x63, 0x2d, 0x64, 0x65, 0x76, 0x2f, 0x6e, 0x65, 0x6f, 0x66, 0x73, 0x2d, 0x6e, 0x6f, 0x64, 0x65,
+	0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2f, 0x63, 0x6f,
+	0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
 }
 
 var (
diff --git a/pkg/services/control/types.proto b/pkg/services/control/types.proto
index f9c8cef2..0729e4eb 100644
--- a/pkg/services/control/types.proto
+++ b/pkg/services/control/types.proto
@@ -157,4 +157,7 @@ enum ShardMode {
 
     // Degraded.
     DEGRADED = 3;
+
+    // DegradedReadOnly.
+    DEGRADED_READ_ONLY = 4;
 }