mkadilov/frostfs-node
1
0
Fork 0
forked from TrueCloudLab/frostfs-node
Code Pull requests Activity

[#1559] local_object_storage: Move shard to the DegradedReadOnly mode

Browse source 
`Degraded` mode can be set by the administrator if needed.
Modifying operations in this mode can lead node into an inconsistent state
because metabase checks such as lock checking are not performed.

Signed-off-by: Evgenii Stratonikov <evgeniy@nspcc.ru>
This commit is contained in:
Evgenii Stratonikov 2022-07-18 14:44:29 +03:00 • committed by fyrchik
parent 3df62769c0
commit 4944490ffb
18 changed files with 58 additions and 32 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
cmd/neofs-adm/internal/modules/morph/internal/types.pb.go generated
View file

@ -1,7 +1,7 @@
// Code generated by protoc-gen-go. DO NOT EDIT.
// versions:
// protoc-gen-go v1.26.0
// protoc v3.19.4
// protoc v3.21.2
// source: cmd/neofs-adm/internal/modules/morph/internal/types.proto
package internal

2
cmd/neofs-cli/modules/control/shards_list.go
View file

@ -109,6 +109,8 @@ func shardModeToString(m control.ShardMode) string {
return "read-only"
case control.ShardMode_DEGRADED:
return "degraded"
case control.ShardMode_DEGRADED_READ_ONLY:
return "degraded-read-only"
default:
return "unknown"
}

9
cmd/neofs-cli/modules/control/shards_set_mode.go
View file

@ -17,9 +17,10 @@ const (
shardIDFlag = "id"
shardClearErrorsFlag = "clear-errors"
shardModeReadOnly = "read-only"
shardModeReadWrite = "read-write"
shardModeDegraded = "degraded"
shardModeReadOnly = "read-only"
shardModeReadWrite = "read-write"
shardModeDegraded = "degraded"
shardModeDegradedReadOnly = "degraded-read-only"
)
var setShardModeCmd = &cobra.Command{
@ -60,6 +61,8 @@ func setShardMode(cmd *cobra.Command, _ []string) {
mode = control.ShardMode_READ_ONLY
case shardModeDegraded:
mode = control.ShardMode_DEGRADED
case shardModeDegradedReadOnly:
mode = control.ShardMode_DEGRADED_READ_ONLY
}
req := new(control.SetShardModeRequest)

2
cmd/neofs-node/config/engine/shard/config.go
View file

@ -88,6 +88,8 @@ func (x *Config) Mode() (m mode.Mode) {
m = mode.ReadOnly
case "degraded":
m = mode.Degraded
case "degraded-read-only":
m = mode.DegradedReadOnly
default:
panic(fmt.Sprintf("unknown shard mode: %s", s))
}

2
pkg/local_object_storage/engine/engine.go
View file

@ -51,7 +51,7 @@ func (e *StorageEngine) reportShardError(
return
}
err = sh.SetMode(mode.Degraded)
err = sh.SetMode(mode.DegradedReadOnly)
if err != nil {
e.log.Error("failed to move shard in degraded mode",
zap.Uint32("error count", errCount),

4
pkg/local_object_storage/engine/error_test.go
View file

@ -125,7 +125,7 @@ func TestErrorReporting(t *testing.T) {
for i := uint32(0); i < 2; i++ {
_, err = e.Get(GetPrm{addr: object.AddressOf(obj)})
require.Error(t, err)
checkShardState(t, e, id[0], errThreshold+i, mode.Degraded)
checkShardState(t, e, id[0], errThreshold+i, mode.DegradedReadOnly)
checkShardState(t, e, id[1], 0, mode.ReadWrite)
}
@ -193,7 +193,7 @@ func TestBlobstorFailback(t *testing.T) {
require.ErrorAs(t, err, &apistatus.ObjectOutOfRange{})
}
checkShardState(t, e, id[0], 1, mode.Degraded)
checkShardState(t, e, id[0], 1, mode.DegradedReadOnly)
checkShardState(t, e, id[1], 0, mode.ReadWrite)
}

8
pkg/local_object_storage/metabase/mode.go
View file

@ -23,12 +23,12 @@ func (db *DB) SetMode(m mode.Mode) error {
}
var err error
switch m {
case mode.Degraded:
switch {
case m.NoMetabase():
db.boltDB = nil
case mode.ReadOnly:
case m.ReadOnly():
err = db.Open(true)
case mode.ReadWrite:
default:
err = db.Open(false)
}
if err == nil && !m.NoMetabase() && !m.ReadOnly() {

3
pkg/local_object_storage/shard/get.go
View file

@ -6,7 +6,6 @@ import (
"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobovnicza"
"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor"
meta "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/metabase"
"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/shard/mode"
"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/writecache"
apistatus "github.com/nspcc-dev/neofs-sdk-go/client/status"
objectSDK "github.com/nspcc-dev/neofs-sdk-go/object"
@ -123,7 +122,7 @@ func (s *Shard) fetchObjectData(addr oid.Address, skipMeta bool, big, small stor
mPrm.SetAddress(addr)
mRes, err := s.metaBase.Exists(mPrm)
if err != nil && s.GetMode() != mode.Degraded {
if err != nil && !s.GetMode().NoMetabase() {
return res, false, err
}
exists = mRes.Exists()

22
pkg/local_object_storage/shard/mode/mode.go
View file

@ -3,19 +3,25 @@ package mode
// Mode represents enumeration of Shard work modes.
type Mode uint32
// ReadWrite is a Mode value for shard that is available
// for read and write operations. Default shard mode.
const ReadWrite Mode = 0
const (
// ReadWrite is a Mode value for shard that is available
// for read and write operations. Default shard mode.
ReadWrite Mode = 0
// DegradedReadOnly is a Mode value for shard that is set automatically
// after a certain number of errors is encountered. It is the same as
// `mode.ReadOnly` but also enables fallback algorithms for getting object
// in case metabase is corrupted.
DegradedReadOnly = Degraded | ReadOnly
)
const (
// ReadOnly is a Mode value for shard that does not
// accept write operation but is readable.
ReadOnly Mode = 1 << iota
// Degraded is a Mode value for shard that is set automatically
// after a certain number of errors is encountered. It is the same as
// `mode.ReadOnly` but also enables fallback algorithms for getting object
// in case metabase is corrupted.
// Degraded is a Mode value for shard when the metabase is unavailable.
// It is hard to perform some modifying operations in this mode, thus it can only be set by an administrator.
Degraded
)
@ -29,6 +35,8 @@ func (m Mode) String() string {
return "READ_ONLY"
case Degraded:
return "DEGRADED"
case DegradedReadOnly:
return "DEGRADED_READ_ONLY"
}
}

2
pkg/services/control/ir/service.pb.go generated
View file

@ -1,7 +1,7 @@
// Code generated by protoc-gen-go. DO NOT EDIT.
// versions:
// protoc-gen-go v1.26.0
// protoc v3.21.1
// protoc v3.21.2
// source: pkg/services/control/ir/service.proto
package control

2
pkg/services/control/ir/service_grpc.pb.go generated
View file

@ -1,7 +1,7 @@
// Code generated by protoc-gen-go-grpc. DO NOT EDIT.
// versions:
// - protoc-gen-go-grpc v1.2.0
// - protoc v3.21.1
// - protoc v3.21.2
// source: pkg/services/control/ir/service.proto
package control

2
pkg/services/control/ir/types.pb.go generated
View file

@ -1,7 +1,7 @@
// Code generated by protoc-gen-go. DO NOT EDIT.
// versions:
// protoc-gen-go v1.26.0
// protoc v3.21.1
// protoc v3.21.2
// source: pkg/services/control/ir/types.proto
package control

2
pkg/services/control/server/list_shards.go
View file

@ -43,6 +43,8 @@ func (s *Server) ListShards(_ context.Context, req *control.ListShardsRequest) (
m = control.ShardMode_READ_ONLY
case mode.Degraded:
m = control.ShardMode_DEGRADED
case mode.DegradedReadOnly:
m = control.ShardMode_DEGRADED_READ_ONLY
default:
m = control.ShardMode_SHARD_MODE_UNDEFINED
}

2
pkg/services/control/server/set_shard_mode.go
View file

@ -32,6 +32,8 @@ func (s *Server) SetShardMode(_ context.Context, req *control.SetShardModeReques
m = mode.ReadOnly
case control.ShardMode_DEGRADED:
m = mode.Degraded
case control.ShardMode_DEGRADED_READ_ONLY:
m = mode.ReadOnly
default:
return nil, status.Error(codes.Internal, fmt.Sprintf("unknown shard mode: %s", requestedMode))
}

2
pkg/services/control/service.pb.go generated
View file

@ -1,7 +1,7 @@
// Code generated by protoc-gen-go. DO NOT EDIT.
// versions:
// protoc-gen-go v1.26.0
// protoc v3.21.1
// protoc v3.21.2
// source: pkg/services/control/service.proto
package control

2
pkg/services/control/service_grpc.pb.go generated
View file

@ -1,7 +1,7 @@
// Code generated by protoc-gen-go-grpc. DO NOT EDIT.
// versions:
// - protoc-gen-go-grpc v1.2.0
// - protoc v3.21.1
// - protoc v3.21.2
// source: pkg/services/control/service.proto
package control

19
pkg/services/control/types.pb.go generated
View file

@ -1,7 +1,7 @@
// Code generated by protoc-gen-go. DO NOT EDIT.
// versions:
// protoc-gen-go v1.26.0
// protoc v3.21.1
// protoc v3.21.2
// source: pkg/services/control/types.proto
package control
@ -146,6 +146,8 @@ const (
ShardMode_READ_ONLY ShardMode = 2
// Degraded.
ShardMode_DEGRADED ShardMode = 3
// DegradedReadOnly.
ShardMode_DEGRADED_READ_ONLY ShardMode = 4
)
// Enum value maps for ShardMode.
@ -155,12 +157,14 @@ var (
1: "READ_WRITE",
2: "READ_ONLY",
3: "DEGRADED",
4: "DEGRADED_READ_ONLY",
}
ShardMode_value = map[string]int32{
"SHARD_MODE_UNDEFINED": 0,
"READ_WRITE": 1,
"READ_ONLY": 2,
"DEGRADED": 3,
"DEGRADED_READ_ONLY": 4,
}
)
@ -652,16 +656,17 @@ var file_pkg_services_control_types_proto_rawDesc = []byte{
0x4e, 0x45, 0x44, 0x10, 0x00, 0x12, 0x0c, 0x0a, 0x08, 0x53, 0x54, 0x41, 0x52, 0x54, 0x49, 0x4e,
0x47, 0x10, 0x01, 0x12, 0x09, 0x0a, 0x05, 0x52, 0x45, 0x41, 0x44, 0x59, 0x10, 0x02, 0x12, 0x11,
0x0a, 0x0d, 0x53, 0x48, 0x55, 0x54, 0x54, 0x49, 0x4e, 0x47, 0x5f, 0x44, 0x4f, 0x57, 0x4e, 0x10,
0x03, 0x2a, 0x52, 0x0a, 0x09, 0x53, 0x68, 0x61, 0x72, 0x64, 0x4d, 0x6f, 0x64, 0x65, 0x12, 0x18,
0x03, 0x2a, 0x6a, 0x0a, 0x09, 0x53, 0x68, 0x61, 0x72, 0x64, 0x4d, 0x6f, 0x64, 0x65, 0x12, 0x18,
0x0a, 0x14, 0x53, 0x48, 0x41, 0x52, 0x44, 0x5f, 0x4d, 0x4f, 0x44, 0x45, 0x5f, 0x55, 0x4e, 0x44,
0x45, 0x46, 0x49, 0x4e, 0x45, 0x44, 0x10, 0x00, 0x12, 0x0e, 0x0a, 0x0a, 0x52, 0x45, 0x41, 0x44,
0x5f, 0x57, 0x52, 0x49, 0x54, 0x45, 0x10, 0x01, 0x12, 0x0d, 0x0a, 0x09, 0x52, 0x45, 0x41, 0x44,
0x5f, 0x4f, 0x4e, 0x4c, 0x59, 0x10, 0x02, 0x12, 0x0c, 0x0a, 0x08, 0x44, 0x45, 0x47, 0x52, 0x41,
0x44, 0x45, 0x44, 0x10, 0x03, 0x42, 0x36, 0x5a, 0x34, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e,
0x63, 0x6f, 0x6d, 0x2f, 0x6e, 0x73, 0x70, 0x63, 0x63, 0x2d, 0x64, 0x65, 0x76, 0x2f, 0x6e, 0x65,
0x6f, 0x66, 0x73, 0x2d, 0x6e, 0x6f, 0x64, 0x65, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x73, 0x65, 0x72,
0x76, 0x69, 0x63, 0x65, 0x73, 0x2f, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x62, 0x06, 0x70,
0x72, 0x6f, 0x74, 0x6f, 0x33,
0x44, 0x45, 0x44, 0x10, 0x03, 0x12, 0x16, 0x0a, 0x12, 0x44, 0x45, 0x47, 0x52, 0x41, 0x44, 0x45,
0x44, 0x5f, 0x52, 0x45, 0x41, 0x44, 0x5f, 0x4f, 0x4e, 0x4c, 0x59, 0x10, 0x04, 0x42, 0x36, 0x5a,
0x34, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x6e, 0x73, 0x70, 0x63,
0x63, 0x2d, 0x64, 0x65, 0x76, 0x2f, 0x6e, 0x65, 0x6f, 0x66, 0x73, 0x2d, 0x6e, 0x6f, 0x64, 0x65,
0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2f, 0x63, 0x6f,
0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
}
var (

3
pkg/services/control/types.proto
View file

@ -157,4 +157,7 @@ enum ShardMode {
// Degraded.
DEGRADED = 3;
// DegradedReadOnly.
DEGRADED_READ_ONLY = 4;
}