[#1559] local_object_storage: Move shard to the DegradedReadOnly mode

`Degraded` mode can be set by the administrator if needed.
Modifying operations in this mode can lead node into an inconsistent state
because metabase checks such as lock checking are not performed.

Signed-off-by: Evgenii Stratonikov <evgeniy@nspcc.ru>
This commit is contained in:
Evgenii Stratonikov 2022-07-18 14:44:29 +03:00 committed by fyrchik
parent 3df62769c0
commit 4944490ffb
18 changed files with 40 additions and 19 deletions

View file

@ -109,6 +109,8 @@ func shardModeToString(m control.ShardMode) string {
return "read-only"
case control.ShardMode_DEGRADED:
return "degraded"
case control.ShardMode_DEGRADED_READ_ONLY:
return "degraded-read-only"
default:
return "unknown"
}

View file

@ -20,6 +20,7 @@ const (
shardModeReadOnly = "read-only"
shardModeReadWrite = "read-write"
shardModeDegraded = "degraded"
shardModeDegradedReadOnly = "degraded-read-only"
)
var setShardModeCmd = &cobra.Command{
@ -60,6 +61,8 @@ func setShardMode(cmd *cobra.Command, _ []string) {
mode = control.ShardMode_READ_ONLY
case shardModeDegraded:
mode = control.ShardMode_DEGRADED
case shardModeDegradedReadOnly:
mode = control.ShardMode_DEGRADED_READ_ONLY
}
req := new(control.SetShardModeRequest)

View file

@ -88,6 +88,8 @@ func (x *Config) Mode() (m mode.Mode) {
m = mode.ReadOnly
case "degraded":
m = mode.Degraded
case "degraded-read-only":
m = mode.DegradedReadOnly
default:
panic(fmt.Sprintf("unknown shard mode: %s", s))
}

View file

@ -51,7 +51,7 @@ func (e *StorageEngine) reportShardError(
return
}
err = sh.SetMode(mode.Degraded)
err = sh.SetMode(mode.DegradedReadOnly)
if err != nil {
e.log.Error("failed to move shard in degraded mode",
zap.Uint32("error count", errCount),

View file

@ -125,7 +125,7 @@ func TestErrorReporting(t *testing.T) {
for i := uint32(0); i < 2; i++ {
_, err = e.Get(GetPrm{addr: object.AddressOf(obj)})
require.Error(t, err)
checkShardState(t, e, id[0], errThreshold+i, mode.Degraded)
checkShardState(t, e, id[0], errThreshold+i, mode.DegradedReadOnly)
checkShardState(t, e, id[1], 0, mode.ReadWrite)
}
@ -193,7 +193,7 @@ func TestBlobstorFailback(t *testing.T) {
require.ErrorAs(t, err, &apistatus.ObjectOutOfRange{})
}
checkShardState(t, e, id[0], 1, mode.Degraded)
checkShardState(t, e, id[0], 1, mode.DegradedReadOnly)
checkShardState(t, e, id[1], 0, mode.ReadWrite)
}

View file

@ -23,12 +23,12 @@ func (db *DB) SetMode(m mode.Mode) error {
}
var err error
switch m {
case mode.Degraded:
switch {
case m.NoMetabase():
db.boltDB = nil
case mode.ReadOnly:
case m.ReadOnly():
err = db.Open(true)
case mode.ReadWrite:
default:
err = db.Open(false)
}
if err == nil && !m.NoMetabase() && !m.ReadOnly() {

View file

@ -6,7 +6,6 @@ import (
"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobovnicza"
"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor"
meta "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/metabase"
"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/shard/mode"
"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/writecache"
apistatus "github.com/nspcc-dev/neofs-sdk-go/client/status"
objectSDK "github.com/nspcc-dev/neofs-sdk-go/object"
@ -123,7 +122,7 @@ func (s *Shard) fetchObjectData(addr oid.Address, skipMeta bool, big, small stor
mPrm.SetAddress(addr)
mRes, err := s.metaBase.Exists(mPrm)
if err != nil && s.GetMode() != mode.Degraded {
if err != nil && !s.GetMode().NoMetabase() {
return res, false, err
}
exists = mRes.Exists()

View file

@ -3,19 +3,25 @@ package mode
// Mode represents enumeration of Shard work modes.
type Mode uint32
const (
// ReadWrite is a Mode value for shard that is available
// for read and write operations. Default shard mode.
const ReadWrite Mode = 0
ReadWrite Mode = 0
// DegradedReadOnly is a Mode value for shard that is set automatically
// after a certain number of errors is encountered. It is the same as
// `mode.ReadOnly` but also enables fallback algorithms for getting object
// in case metabase is corrupted.
DegradedReadOnly = Degraded | ReadOnly
)
const (
// ReadOnly is a Mode value for shard that does not
// accept write operation but is readable.
ReadOnly Mode = 1 << iota
// Degraded is a Mode value for shard that is set automatically
// after a certain number of errors is encountered. It is the same as
// `mode.ReadOnly` but also enables fallback algorithms for getting object
// in case metabase is corrupted.
// Degraded is a Mode value for shard when the metabase is unavailable.
// It is hard to perform some modifying operations in this mode, thus it can only be set by an administrator.
Degraded
)
@ -29,6 +35,8 @@ func (m Mode) String() string {
return "READ_ONLY"
case Degraded:
return "DEGRADED"
case DegradedReadOnly:
return "DEGRADED_READ_ONLY"
}
}

Binary file not shown.

Binary file not shown.

Binary file not shown.

View file

@ -43,6 +43,8 @@ func (s *Server) ListShards(_ context.Context, req *control.ListShardsRequest) (
m = control.ShardMode_READ_ONLY
case mode.Degraded:
m = control.ShardMode_DEGRADED
case mode.DegradedReadOnly:
m = control.ShardMode_DEGRADED_READ_ONLY
default:
m = control.ShardMode_SHARD_MODE_UNDEFINED
}

View file

@ -32,6 +32,8 @@ func (s *Server) SetShardMode(_ context.Context, req *control.SetShardModeReques
m = mode.ReadOnly
case control.ShardMode_DEGRADED:
m = mode.Degraded
case control.ShardMode_DEGRADED_READ_ONLY:
m = mode.ReadOnly
default:
return nil, status.Error(codes.Internal, fmt.Sprintf("unknown shard mode: %s", requestedMode))
}

Binary file not shown.

Binary file not shown.

Binary file not shown.

View file

@ -157,4 +157,7 @@ enum ShardMode {
// Degraded.
DEGRADED = 3;
// DegradedReadOnly.
DEGRADED_READ_ONLY = 4;
}