[#1559] local_object_storage: Move shard to the DegradedReadOnly mode

`Degraded` mode can be set by the administrator if needed.
Modifying operations in this mode can lead node into an inconsistent state
because metabase checks such as lock checking are not performed.

Signed-off-by: Evgenii Stratonikov <evgeniy@nspcc.ru>

cmd/neofs-cli/modules/control/shards_list.go

@@ -109,6 +109,8 @@ func shardModeToString(m control.ShardMode) string {
 		return "read-only"
 	case control.ShardMode_DEGRADED:
 		return "degraded"
+	case control.ShardMode_DEGRADED_READ_ONLY:
+		return "degraded-read-only"
 	default:
 		return "unknown"
 	}

cmd/neofs-cli/modules/control/shards_set_mode.go

@@ -20,6 +20,7 @@ const (
 	shardModeReadOnly         = "read-only"
 	shardModeReadWrite        = "read-write"
 	shardModeDegraded         = "degraded"
+	shardModeDegradedReadOnly = "degraded-read-only"
 )
 
 var setShardModeCmd = &cobra.Command{
@@ -60,6 +61,8 @@ func setShardMode(cmd *cobra.Command, _ []string) {
 		mode = control.ShardMode_READ_ONLY
 	case shardModeDegraded:
 		mode = control.ShardMode_DEGRADED
+	case shardModeDegradedReadOnly:
+		mode = control.ShardMode_DEGRADED_READ_ONLY
 	}
 
 	req := new(control.SetShardModeRequest)

cmd/neofs-node/config/engine/shard/config.go

@@ -88,6 +88,8 @@ func (x *Config) Mode() (m mode.Mode) {
 		m = mode.ReadOnly
 	case "degraded":
 		m = mode.Degraded
+	case "degraded-read-only":
+		m = mode.DegradedReadOnly
 	default:
 		panic(fmt.Sprintf("unknown shard mode: %s", s))
 	}

pkg/local_object_storage/engine/engine.go

@@ -51,7 +51,7 @@ func (e *StorageEngine) reportShardError(
 		return
 	}
 
-	err = sh.SetMode(mode.Degraded)
+	err = sh.SetMode(mode.DegradedReadOnly)
 	if err != nil {
 		e.log.Error("failed to move shard in degraded mode",
 			zap.Uint32("error count", errCount),

pkg/local_object_storage/engine/error_test.go

@@ -125,7 +125,7 @@ func TestErrorReporting(t *testing.T) {
 		for i := uint32(0); i < 2; i++ {
 			_, err = e.Get(GetPrm{addr: object.AddressOf(obj)})
 			require.Error(t, err)
-			checkShardState(t, e, id[0], errThreshold+i, mode.Degraded)
+			checkShardState(t, e, id[0], errThreshold+i, mode.DegradedReadOnly)
 			checkShardState(t, e, id[1], 0, mode.ReadWrite)
 		}
 
@@ -193,7 +193,7 @@ func TestBlobstorFailback(t *testing.T) {
 		require.ErrorAs(t, err, &apistatus.ObjectOutOfRange{})
 	}
 
-	checkShardState(t, e, id[0], 1, mode.Degraded)
+	checkShardState(t, e, id[0], 1, mode.DegradedReadOnly)
 	checkShardState(t, e, id[1], 0, mode.ReadWrite)
 }
 

pkg/local_object_storage/metabase/mode.go

@@ -23,12 +23,12 @@ func (db *DB) SetMode(m mode.Mode) error {
 	}
 
 	var err error
-	switch m {
+	switch {
-	case mode.Degraded:
+	case m.NoMetabase():
 		db.boltDB = nil
-	case mode.ReadOnly:
+	case m.ReadOnly():
 		err = db.Open(true)
-	case mode.ReadWrite:
+	default:
 		err = db.Open(false)
 	}
 	if err == nil && !m.NoMetabase() && !m.ReadOnly() {

pkg/local_object_storage/shard/get.go

@@ -6,7 +6,6 @@ import (
 	"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobovnicza"
 	"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor"
 	meta "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/metabase"
-	"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/shard/mode"
 	"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/writecache"
 	apistatus "github.com/nspcc-dev/neofs-sdk-go/client/status"
 	objectSDK "github.com/nspcc-dev/neofs-sdk-go/object"
@@ -123,7 +122,7 @@ func (s *Shard) fetchObjectData(addr oid.Address, skipMeta bool, big, small stor
 		mPrm.SetAddress(addr)
 
 		mRes, err := s.metaBase.Exists(mPrm)
-		if err != nil && s.GetMode() != mode.Degraded {
+		if err != nil && !s.GetMode().NoMetabase() {
 			return res, false, err
 		}
 		exists = mRes.Exists()

pkg/local_object_storage/shard/mode/mode.go

@@ -3,19 +3,25 @@ package mode
 // Mode represents enumeration of Shard work modes.
 type Mode uint32
 
+const (
 	// ReadWrite is a Mode value for shard that is available
 	// for read and write operations. Default shard mode.
-const ReadWrite Mode = 0
+	ReadWrite Mode = 0
+
+	// DegradedReadOnly is a Mode value for shard that is set automatically
+	// after a certain number of errors is encountered. It is the same as
+	// `mode.ReadOnly` but also enables fallback algorithms for getting object
+	// in case metabase is corrupted.
+	DegradedReadOnly = Degraded | ReadOnly
+)
 
 const (
 	// ReadOnly is a Mode value for shard that does not
 	// accept write operation but is readable.
 	ReadOnly Mode = 1 << iota
 
-	// Degraded is a Mode value for shard that is set automatically
+	// Degraded is a Mode value for shard when the metabase is unavailable.
-	// after a certain number of errors is encountered. It is the same as
+	// It is hard to perform some modifying operations in this mode, thus it can only be set by an administrator.
-	// `mode.ReadOnly` but also enables fallback algorithms for getting object
-	// in case metabase is corrupted.
 	Degraded
 )
 
@@ -29,6 +35,8 @@ func (m Mode) String() string {
 		return "READ_ONLY"
 	case Degraded:
 		return "DEGRADED"
+	case DegradedReadOnly:
+		return "DEGRADED_READ_ONLY"
 	}
 }
 

pkg/services/control/server/list_shards.go

@@ -43,6 +43,8 @@ func (s *Server) ListShards(_ context.Context, req *control.ListShardsRequest) (
 			m = control.ShardMode_READ_ONLY
 		case mode.Degraded:
 			m = control.ShardMode_DEGRADED
+		case mode.DegradedReadOnly:
+			m = control.ShardMode_DEGRADED_READ_ONLY
 		default:
 			m = control.ShardMode_SHARD_MODE_UNDEFINED
 		}

pkg/services/control/server/set_shard_mode.go

@@ -32,6 +32,8 @@ func (s *Server) SetShardMode(_ context.Context, req *control.SetShardModeReques
 		m = mode.ReadOnly
 	case control.ShardMode_DEGRADED:
 		m = mode.Degraded
+	case control.ShardMode_DEGRADED_READ_ONLY:
+		m = mode.ReadOnly
 	default:
 		return nil, status.Error(codes.Internal, fmt.Sprintf("unknown shard mode: %s", requestedMode))
 	}

pkg/services/control/types.proto

@@ -157,4 +157,7 @@ enum ShardMode {
 
     // Degraded.
     DEGRADED = 3;
+
+    // DegradedReadOnly.
+    DEGRADED_READ_ONLY = 4;
 }