[#1559] local_object_storage: Move shard to the DegradedReadOnly mode

`Degraded` mode can be set by the administrator if needed.
Modifying operations in this mode can lead node into an inconsistent state
because metabase checks such as lock checking are not performed.

Signed-off-by: Evgenii Stratonikov <evgeniy@nspcc.ru>
This commit is contained in:
Evgenii Stratonikov 2022-07-18 14:44:29 +03:00 committed by fyrchik
parent 3df62769c0
commit 4944490ffb
18 changed files with 40 additions and 19 deletions

View file

@ -109,6 +109,8 @@ func shardModeToString(m control.ShardMode) string {
return "read-only" return "read-only"
case control.ShardMode_DEGRADED: case control.ShardMode_DEGRADED:
return "degraded" return "degraded"
case control.ShardMode_DEGRADED_READ_ONLY:
return "degraded-read-only"
default: default:
return "unknown" return "unknown"
} }

View file

@ -20,6 +20,7 @@ const (
shardModeReadOnly = "read-only" shardModeReadOnly = "read-only"
shardModeReadWrite = "read-write" shardModeReadWrite = "read-write"
shardModeDegraded = "degraded" shardModeDegraded = "degraded"
shardModeDegradedReadOnly = "degraded-read-only"
) )
var setShardModeCmd = &cobra.Command{ var setShardModeCmd = &cobra.Command{
@ -60,6 +61,8 @@ func setShardMode(cmd *cobra.Command, _ []string) {
mode = control.ShardMode_READ_ONLY mode = control.ShardMode_READ_ONLY
case shardModeDegraded: case shardModeDegraded:
mode = control.ShardMode_DEGRADED mode = control.ShardMode_DEGRADED
case shardModeDegradedReadOnly:
mode = control.ShardMode_DEGRADED_READ_ONLY
} }
req := new(control.SetShardModeRequest) req := new(control.SetShardModeRequest)

View file

@ -88,6 +88,8 @@ func (x *Config) Mode() (m mode.Mode) {
m = mode.ReadOnly m = mode.ReadOnly
case "degraded": case "degraded":
m = mode.Degraded m = mode.Degraded
case "degraded-read-only":
m = mode.DegradedReadOnly
default: default:
panic(fmt.Sprintf("unknown shard mode: %s", s)) panic(fmt.Sprintf("unknown shard mode: %s", s))
} }

View file

@ -51,7 +51,7 @@ func (e *StorageEngine) reportShardError(
return return
} }
err = sh.SetMode(mode.Degraded) err = sh.SetMode(mode.DegradedReadOnly)
if err != nil { if err != nil {
e.log.Error("failed to move shard in degraded mode", e.log.Error("failed to move shard in degraded mode",
zap.Uint32("error count", errCount), zap.Uint32("error count", errCount),

View file

@ -125,7 +125,7 @@ func TestErrorReporting(t *testing.T) {
for i := uint32(0); i < 2; i++ { for i := uint32(0); i < 2; i++ {
_, err = e.Get(GetPrm{addr: object.AddressOf(obj)}) _, err = e.Get(GetPrm{addr: object.AddressOf(obj)})
require.Error(t, err) require.Error(t, err)
checkShardState(t, e, id[0], errThreshold+i, mode.Degraded) checkShardState(t, e, id[0], errThreshold+i, mode.DegradedReadOnly)
checkShardState(t, e, id[1], 0, mode.ReadWrite) checkShardState(t, e, id[1], 0, mode.ReadWrite)
} }
@ -193,7 +193,7 @@ func TestBlobstorFailback(t *testing.T) {
require.ErrorAs(t, err, &apistatus.ObjectOutOfRange{}) require.ErrorAs(t, err, &apistatus.ObjectOutOfRange{})
} }
checkShardState(t, e, id[0], 1, mode.Degraded) checkShardState(t, e, id[0], 1, mode.DegradedReadOnly)
checkShardState(t, e, id[1], 0, mode.ReadWrite) checkShardState(t, e, id[1], 0, mode.ReadWrite)
} }

View file

@ -23,12 +23,12 @@ func (db *DB) SetMode(m mode.Mode) error {
} }
var err error var err error
switch m { switch {
case mode.Degraded: case m.NoMetabase():
db.boltDB = nil db.boltDB = nil
case mode.ReadOnly: case m.ReadOnly():
err = db.Open(true) err = db.Open(true)
case mode.ReadWrite: default:
err = db.Open(false) err = db.Open(false)
} }
if err == nil && !m.NoMetabase() && !m.ReadOnly() { if err == nil && !m.NoMetabase() && !m.ReadOnly() {

View file

@ -6,7 +6,6 @@ import (
"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobovnicza" "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobovnicza"
"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor" "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor"
meta "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/metabase" meta "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/metabase"
"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/shard/mode"
"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/writecache" "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/writecache"
apistatus "github.com/nspcc-dev/neofs-sdk-go/client/status" apistatus "github.com/nspcc-dev/neofs-sdk-go/client/status"
objectSDK "github.com/nspcc-dev/neofs-sdk-go/object" objectSDK "github.com/nspcc-dev/neofs-sdk-go/object"
@ -123,7 +122,7 @@ func (s *Shard) fetchObjectData(addr oid.Address, skipMeta bool, big, small stor
mPrm.SetAddress(addr) mPrm.SetAddress(addr)
mRes, err := s.metaBase.Exists(mPrm) mRes, err := s.metaBase.Exists(mPrm)
if err != nil && s.GetMode() != mode.Degraded { if err != nil && !s.GetMode().NoMetabase() {
return res, false, err return res, false, err
} }
exists = mRes.Exists() exists = mRes.Exists()

View file

@ -3,19 +3,25 @@ package mode
// Mode represents enumeration of Shard work modes. // Mode represents enumeration of Shard work modes.
type Mode uint32 type Mode uint32
const (
// ReadWrite is a Mode value for shard that is available // ReadWrite is a Mode value for shard that is available
// for read and write operations. Default shard mode. // for read and write operations. Default shard mode.
const ReadWrite Mode = 0 ReadWrite Mode = 0
// DegradedReadOnly is a Mode value for shard that is set automatically
// after a certain number of errors is encountered. It is the same as
// `mode.ReadOnly` but also enables fallback algorithms for getting object
// in case metabase is corrupted.
DegradedReadOnly = Degraded | ReadOnly
)
const ( const (
// ReadOnly is a Mode value for shard that does not // ReadOnly is a Mode value for shard that does not
// accept write operation but is readable. // accept write operation but is readable.
ReadOnly Mode = 1 << iota ReadOnly Mode = 1 << iota
// Degraded is a Mode value for shard that is set automatically // Degraded is a Mode value for shard when the metabase is unavailable.
// after a certain number of errors is encountered. It is the same as // It is hard to perform some modifying operations in this mode, thus it can only be set by an administrator.
// `mode.ReadOnly` but also enables fallback algorithms for getting object
// in case metabase is corrupted.
Degraded Degraded
) )
@ -29,6 +35,8 @@ func (m Mode) String() string {
return "READ_ONLY" return "READ_ONLY"
case Degraded: case Degraded:
return "DEGRADED" return "DEGRADED"
case DegradedReadOnly:
return "DEGRADED_READ_ONLY"
} }
} }

Binary file not shown.

Binary file not shown.

Binary file not shown.

View file

@ -43,6 +43,8 @@ func (s *Server) ListShards(_ context.Context, req *control.ListShardsRequest) (
m = control.ShardMode_READ_ONLY m = control.ShardMode_READ_ONLY
case mode.Degraded: case mode.Degraded:
m = control.ShardMode_DEGRADED m = control.ShardMode_DEGRADED
case mode.DegradedReadOnly:
m = control.ShardMode_DEGRADED_READ_ONLY
default: default:
m = control.ShardMode_SHARD_MODE_UNDEFINED m = control.ShardMode_SHARD_MODE_UNDEFINED
} }

View file

@ -32,6 +32,8 @@ func (s *Server) SetShardMode(_ context.Context, req *control.SetShardModeReques
m = mode.ReadOnly m = mode.ReadOnly
case control.ShardMode_DEGRADED: case control.ShardMode_DEGRADED:
m = mode.Degraded m = mode.Degraded
case control.ShardMode_DEGRADED_READ_ONLY:
m = mode.ReadOnly
default: default:
return nil, status.Error(codes.Internal, fmt.Sprintf("unknown shard mode: %s", requestedMode)) return nil, status.Error(codes.Internal, fmt.Sprintf("unknown shard mode: %s", requestedMode))
} }

Binary file not shown.

Binary file not shown.

Binary file not shown.

View file

@ -157,4 +157,7 @@ enum ShardMode {
// Degraded. // Degraded.
DEGRADED = 3; DEGRADED = 3;
// DegradedReadOnly.
DEGRADED_READ_ONLY = 4;
} }