potyarkin/frostfs-node
1
0
Fork 0
forked from TrueCloudLab/frostfs-node
Code Pull requests Activity

[#1143] shard: Introduce explicit Degraded mode

Browse source 
`Degraded` mode is set automatically after error counter is over the
threshold. `ReadOnly` mode can still be set by an administrator.

Signed-off-by: Evgenii Stratonikov <evgeniy@nspcc.ru>
This commit is contained in:
Evgenii Stratonikov 2022-03-17 14:55:25 +03:00 committed by fyrchik
parent 9eb70c18c3
commit 6472a170eb
21 changed files with 62 additions and 24 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

8
cmd/neofs-cli/modules/control.go
View file

@ -78,6 +78,7 @@ const (
shardModeReadOnly = "read-only"
shardModeReadWrite = "read-write"
shardModeDegraded = "degraded"
)
const (
@ -124,9 +125,10 @@ func initControlSetShardModeCmd() {
flags.String(controlRPC, controlRPCDefault, controlRPCUsage)
flags.StringVarP(&shardID, shardIDFlag, "", "", "ID of the shard in base58 encoding")
flags.StringVarP(&shardMode, shardModeFlag, "", "",
fmt.Sprintf("new shard mode keyword ('%s', '%s')",
fmt.Sprintf("new shard mode keyword ('%s', '%s', '%s')",
shardModeReadWrite,
shardModeReadOnly,
shardModeDegraded,
),
)
flags.Bool(shardClearErrorsFlag, false, "Set shard error count to 0")
@ -490,6 +492,8 @@ func prettyPrintShards(cmd *cobra.Command, ii []*control.ShardInfo) {
mode = "read-write"
case control.ShardMode_READ_ONLY:
mode = "read-only"
case control.ShardMode_DEGRADED:
mode = "degraded"
default:
mode = "unknown"
}
@ -526,6 +530,8 @@ func setShardMode(cmd *cobra.Command, _ []string) {
mode = control.ShardMode_READ_WRITE
case shardModeReadOnly:
mode = control.ShardMode_READ_ONLY
case shardModeDegraded:
mode = control.ShardMode_DEGRADED
}
req := new(control.SetShardModeRequest)

2
cmd/neofs-node/config/engine/shard/config.go
View file

@ -77,6 +77,8 @@ func (x *Config) Mode() (m shard.Mode) {
m = shard.ModeReadWrite
case "read-only":
m = shard.ModeReadOnly
case "degraded":
m = shard.ModeDegraded
default:
panic(fmt.Sprintf("unknown shard mode: %s", s))
}

6
pkg/local_object_storage/engine/engine.go
View file

@ -50,13 +50,13 @@ func (e *StorageEngine) reportShardError(
return
}
err = sh.SetMode(shard.ModeReadOnly)
err = sh.SetMode(shard.ModeDegraded)
if err != nil {
e.log.Error("failed to move shard in read-only mode",
e.log.Error("failed to move shard in degraded mode",
zap.Uint32("error count", errCount),
zap.Error(err))
} else {
e.log.Info("shard is moved in read-only due to error threshold",
e.log.Info("shard is moved in degraded mode due to error threshold",
zap.Stringer("shard_id", sh.ID()),
zap.Uint32("error count", errCount))
}

4
pkg/local_object_storage/engine/error_test.go
View file

@ -118,7 +118,7 @@ func TestErrorReporting(t *testing.T) {
for i := uint32(0); i < 2; i++ {
_, err = e.Get(&GetPrm{addr: object.AddressOf(obj)})
require.Error(t, err)
checkShardState(t, e, id[0], errThreshold+i, shard.ModeReadOnly)
checkShardState(t, e, id[0], errThreshold+i, shard.ModeDegraded)
checkShardState(t, e, id[1], 0, shard.ModeReadWrite)
}
@ -185,7 +185,7 @@ func TestBlobstorFailback(t *testing.T) {
require.True(t, errors.Is(err, object.ErrRangeOutOfBounds), "got: %v", err)
}
checkShardState(t, e, id[0], 4, shard.ModeReadOnly)
checkShardState(t, e, id[0], 4, shard.ModeDegraded)
checkShardState(t, e, id[1], 0, shard.ModeReadWrite)
}

2
pkg/local_object_storage/shard/delete.go
View file

@ -31,7 +31,7 @@ func (p *DeletePrm) WithAddresses(addr ...*addressSDK.Address) *DeletePrm {
// Delete removes data from the shard's writeCache, metaBase and
// blobStor.
func (s *Shard) Delete(prm *DeletePrm) (*DeleteRes, error) {
if s.GetMode() == ModeReadOnly {
if s.GetMode() != ModeReadWrite {
return nil, ErrReadOnlyMode
}

2
pkg/local_object_storage/shard/gc.go
View file

@ -174,7 +174,7 @@ func (gc *gc) stop() {
// with GC-marked graves.
// Does nothing if shard is in "read-only" mode.
func (s *Shard) removeGarbage() {
if s.GetMode() == ModeReadOnly {
if s.GetMode() != ModeReadWrite {
return
}

2
pkg/local_object_storage/shard/inhume.go
View file

@ -51,7 +51,7 @@ func (p *InhumePrm) MarkAsGarbage(addr ...*addressSDK.Address) *InhumePrm {
//
// Returns ErrReadOnlyMode error if shard is in "read-only" mode.
func (s *Shard) Inhume(prm *InhumePrm) (*InhumeRes, error) {
if s.GetMode() == ModeReadOnly {
if s.GetMode() != ModeReadWrite {
return nil, ErrReadOnlyMode
}

2
pkg/local_object_storage/shard/lock.go
View file

@ -14,7 +14,7 @@ import (
//
// Locked list should be unique. Panics if it is empty.
func (s *Shard) Lock(idCnr cid.ID, locker oid.ID, locked []oid.ID) error {
if s.GetMode() == ModeReadOnly {
if s.GetMode() != ModeReadWrite {
return ErrReadOnlyMode
}

10
pkg/local_object_storage/shard/mode.go
View file

@ -21,6 +21,12 @@ const (
// ModeReadOnly is a Mode value for shard that does not
// accept write operation but is readable.
ModeReadOnly
// ModeDegraded is a Mode value for shard that is set automatically
// after a certain number of errors is encountered. It is the same as
// `ModeReadOnly` but also enables fallback algorithms for getting object
// in case metabase is corrupted.
ModeDegraded
)
func (m Mode) String() string {
@ -31,6 +37,8 @@ func (m Mode) String() string {
return "READ_WRITE"
case ModeReadOnly:
return "READ_ONLY"
case ModeDegraded:
return "DEGRADED"
}
}
@ -46,6 +54,8 @@ func (s *Shard) SetMode(m Mode) error {
switch m {
case ModeReadOnly:
s.writeCache.SetMode(writecache.ModeReadOnly)
case ModeDegraded:
s.writeCache.SetMode(writecache.ModeDegraded)
case ModeReadWrite:
s.writeCache.SetMode(writecache.ModeReadWrite)
}

2
pkg/local_object_storage/shard/move.go
View file

@ -27,7 +27,7 @@ func (p *ToMoveItPrm) WithAddress(addr *addressSDK.Address) *ToMoveItPrm {
// ToMoveIt calls metabase.ToMoveIt method to mark object as relocatable to
// another shard.
func (s *Shard) ToMoveIt(prm *ToMoveItPrm) (*ToMoveItRes, error) {
if s.GetMode() == ModeReadOnly {
if s.GetMode() != ModeReadWrite {
return nil, ErrReadOnlyMode
}

2
pkg/local_object_storage/shard/put.go
View file

@ -33,7 +33,7 @@ func (p *PutPrm) WithObject(obj *object.Object) *PutPrm {
//
// Returns ErrReadOnlyMode error if shard is in "read-only" mode.
func (s *Shard) Put(prm *PutPrm) (*PutRes, error) {
if s.GetMode() == ModeReadOnly {
if s.GetMode() != ModeReadWrite {
return nil, ErrReadOnlyMode
}

2
pkg/local_object_storage/writecache/delete.go
View file

@ -16,7 +16,7 @@ import (
func (c *cache) Delete(addr *addressSDK.Address) error {
c.modeMtx.RLock()
defer c.modeMtx.RUnlock()
if c.mode == ModeReadOnly {
if c.readOnly() {
return ErrReadOnly
}

4
pkg/local_object_storage/writecache/flush.go
View file

@ -65,7 +65,7 @@ func (c *cache) flush() {
sz := 0
c.modeMtx.RLock()
if c.mode == ModeReadOnly {
if c.readOnly() {
c.modeMtx.RUnlock()
time.Sleep(time.Second)
continue
@ -127,7 +127,7 @@ func (c *cache) flushBigObjects() {
select {
case <-tick.C:
c.modeMtx.RLock()
if c.mode == ModeReadOnly {
if c.readOnly() {
c.modeMtx.RUnlock()
break
}

2
pkg/local_object_storage/writecache/iterate.go
View file

@ -36,7 +36,7 @@ func (p *IterationPrm) WithIgnoreErrors(ignore bool) *IterationPrm {
func (c *cache) Iterate(prm *IterationPrm) error {
c.modeMtx.RLock()
defer c.modeMtx.RUnlock()
if c.mode != ModeReadOnly {
if !c.readOnly() {
return nil
}

9
pkg/local_object_storage/writecache/mode.go
View file

@ -14,6 +14,9 @@ const (
// ModeReadOnly is a mode in which write-cache doesn't flush anything to a metabase.
ModeReadOnly
// ModeDegraded is similar to a shard's degraded mode.
ModeDegraded
)
// ErrReadOnly is returned when Put/Write is performed in a read-only mode.
@ -50,3 +53,9 @@ func (c *cache) SetMode(m Mode) {
time.Sleep(time.Second)
}
}
// readOnly returns true if current mode is read-only.
// `c.modeMtx` must be taken.
func (c *cache) readOnly() bool {
return c.mode != ModeReadWrite
}

2
pkg/local_object_storage/writecache/persist.go
View file

@ -21,7 +21,7 @@ func (c *cache) persistLoop() {
select {
case <-tick.C:
c.modeMtx.RLock()
if c.mode == ModeReadOnly {
if c.readOnly() {
c.modeMtx.RUnlock()
continue
}

2
pkg/local_object_storage/writecache/put.go
View file

@ -15,7 +15,7 @@ var ErrBigObject = errors.New("too big object")
func (c *cache) Put(o *objectSDK.Object) error {
c.modeMtx.RLock()
defer c.modeMtx.RUnlock()
if c.mode == ModeReadOnly {
if c.readOnly() {
return ErrReadOnly
}

2
pkg/services/control/server/list_shards.go
View file

@ -40,6 +40,8 @@ func (s *Server) ListShards(_ context.Context, req *control.ListShardsRequest) (
mode = control.ShardMode_READ_WRITE
case shard.ModeReadOnly:
mode = control.ShardMode_READ_ONLY
case shard.ModeDegraded:
mode = control.ShardMode_DEGRADED
default:
mode = control.ShardMode_SHARD_MODE_UNDEFINED
}

2
pkg/services/control/server/set_shard_mode.go
View file

@ -29,6 +29,8 @@ func (s *Server) SetShardMode(_ context.Context, req *control.SetShardModeReques
mode = shard.ModeReadWrite
case control.ShardMode_READ_ONLY:
mode = shard.ModeReadOnly
case control.ShardMode_DEGRADED:
mode = shard.ModeDegraded
default:
return nil, status.Error(codes.Internal, fmt.Sprintf("unknown shard mode: %s", requestedMode))
}

16
pkg/services/control/types.pb.go generated
View file

@ -144,6 +144,8 @@ const (
ShardMode_READ_WRITE ShardMode = 1
// Read-only.
ShardMode_READ_ONLY ShardMode = 2
// Degraded.
ShardMode_DEGRADED ShardMode = 3
)
// Enum value maps for ShardMode.
@ -152,11 +154,13 @@ var (
0: "SHARD_MODE_UNDEFINED",
1: "READ_WRITE",
2: "READ_ONLY",
3: "DEGRADED",
}
ShardMode_value = map[string]int32{
"SHARD_MODE_UNDEFINED": 0,
"READ_WRITE": 1,
"READ_ONLY": 2,
"DEGRADED": 3,
}
)
@ -636,16 +640,16 @@ var file_pkg_services_control_types_proto_rawDesc = []byte{
0x41, 0x54, 0x55, 0x53, 0x5f, 0x55, 0x4e, 0x44, 0x45, 0x46, 0x49, 0x4e, 0x45, 0x44, 0x10, 0x00,
0x12, 0x0c, 0x0a, 0x08, 0x53, 0x54, 0x41, 0x52, 0x54, 0x49, 0x4e, 0x47, 0x10, 0x01, 0x12, 0x09,
0x0a, 0x05, 0x52, 0x45, 0x41, 0x44, 0x59, 0x10, 0x02, 0x12, 0x11, 0x0a, 0x0d, 0x53, 0x48, 0x55,
0x54, 0x54, 0x49, 0x4e, 0x47, 0x5f, 0x44, 0x4f, 0x57, 0x4e, 0x10, 0x03, 0x2a, 0x44, 0x0a, 0x09,
0x54, 0x54, 0x49, 0x4e, 0x47, 0x5f, 0x44, 0x4f, 0x57, 0x4e, 0x10, 0x03, 0x2a, 0x52, 0x0a, 0x09,
0x53, 0x68, 0x61, 0x72, 0x64, 0x4d, 0x6f, 0x64, 0x65, 0x12, 0x18, 0x0a, 0x14, 0x53, 0x48, 0x41,
0x52, 0x44, 0x5f, 0x4d, 0x4f, 0x44, 0x45, 0x5f, 0x55, 0x4e, 0x44, 0x45, 0x46, 0x49, 0x4e, 0x45,
0x44, 0x10, 0x00, 0x12, 0x0e, 0x0a, 0x0a, 0x52, 0x45, 0x41, 0x44, 0x5f, 0x57, 0x52, 0x49, 0x54,
0x45, 0x10, 0x01, 0x12, 0x0d, 0x0a, 0x09, 0x52, 0x45, 0x41, 0x44, 0x5f, 0x4f, 0x4e, 0x4c, 0x59,
0x10, 0x02, 0x42, 0x36, 0x5a, 0x34, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d,
0x2f, 0x6e, 0x73, 0x70, 0x63, 0x63, 0x2d, 0x64, 0x65, 0x76, 0x2f, 0x6e, 0x65, 0x6f, 0x66, 0x73,
0x2d, 0x6e, 0x6f, 0x64, 0x65, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63,
0x65, 0x73, 0x2f, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74,
0x6f, 0x33,
0x10, 0x02, 0x12, 0x0c, 0x0a, 0x08, 0x44, 0x45, 0x47, 0x52, 0x41, 0x44, 0x45, 0x44, 0x10, 0x03,
0x42, 0x36, 0x5a, 0x34, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x6e,
0x73, 0x70, 0x63, 0x63, 0x2d, 0x64, 0x65, 0x76, 0x2f, 0x6e, 0x65, 0x6f, 0x66, 0x73, 0x2d, 0x6e,
0x6f, 0x64, 0x65, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73,
0x2f, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
}
var (

3
pkg/services/control/types.proto
View file

@ -151,4 +151,7 @@ enum ShardMode {
// Read-only.
READ_ONLY = 2;
// Degraded.
DEGRADED = 3;
}