Reapply "[#446] engine: Move to read-only on blobstor errors" #1172
4 changed files with 26 additions and 18 deletions
|
@ -8,6 +8,7 @@ import (
|
||||||
|
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs"
|
"git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/container"
|
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/container"
|
||||||
|
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/metaerr"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard"
|
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode"
|
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr"
|
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr"
|
||||||
|
@ -49,6 +50,7 @@ type shardWrapper struct {
|
||||||
|
|
||||||
type setModeRequest struct {
|
type setModeRequest struct {
|
||||||
sh *shard.Shard
|
sh *shard.Shard
|
||||||
|
isMeta bool
|
||||||
errorCount uint32
|
errorCount uint32
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -74,7 +76,7 @@ func (e *StorageEngine) setModeLoop() {
|
||||||
if !ok {
|
if !ok {
|
||||||
inProgress[sid] = struct{}{}
|
inProgress[sid] = struct{}{}
|
||||||
go func() {
|
go func() {
|
||||||
e.moveToDegraded(r.sh, r.errorCount)
|
e.moveToDegraded(r.sh, r.errorCount, r.isMeta)
|
||||||
|
|
||||||
mtx.Lock()
|
mtx.Lock()
|
||||||
delete(inProgress, sid)
|
delete(inProgress, sid)
|
||||||
|
@ -86,7 +88,7 @@ func (e *StorageEngine) setModeLoop() {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func (e *StorageEngine) moveToDegraded(sh *shard.Shard, errCount uint32) {
|
func (e *StorageEngine) moveToDegraded(sh *shard.Shard, errCount uint32, isMeta bool) {
|
||||||
sid := sh.ID()
|
sid := sh.ID()
|
||||||
log := e.log.With(
|
log := e.log.With(
|
||||||
zap.Stringer("shard_id", sid),
|
zap.Stringer("shard_id", sid),
|
||||||
|
@ -95,21 +97,23 @@ func (e *StorageEngine) moveToDegraded(sh *shard.Shard, errCount uint32) {
|
||||||
e.mtx.RLock()
|
e.mtx.RLock()
|
||||||
defer e.mtx.RUnlock()
|
defer e.mtx.RUnlock()
|
||||||
|
|
||||||
err := sh.SetMode(mode.DegradedReadOnly)
|
if isMeta {
|
||||||
if err != nil {
|
err := sh.SetMode(mode.DegradedReadOnly)
|
||||||
|
if err == nil {
|
||||||
|
log.Info(logs.EngineShardIsMovedInDegradedModeDueToErrorThreshold)
|
||||||
|
return
|
||||||
|
}
|
||||||
log.Error(logs.EngineFailedToMoveShardInDegradedreadonlyModeMovingToReadonly,
|
log.Error(logs.EngineFailedToMoveShardInDegradedreadonlyModeMovingToReadonly,
|
||||||
zap.Error(err))
|
zap.Error(err))
|
||||||
|
|
||||||
err = sh.SetMode(mode.ReadOnly)
|
|
||||||
if err != nil {
|
|
||||||
log.Error(logs.EngineFailedToMoveShardInReadonlyMode,
|
|
||||||
zap.Error(err))
|
|
||||||
} else {
|
|
||||||
log.Info(logs.EngineShardIsMovedInReadonlyModeDueToErrorThreshold)
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
log.Info(logs.EngineShardIsMovedInDegradedModeDueToErrorThreshold)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
err := sh.SetMode(mode.ReadOnly)
|
||||||
|
if err != nil {
|
||||||
|
log.Error(logs.EngineFailedToMoveShardInReadonlyMode, zap.Error(err))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
log.Info(logs.EngineShardIsMovedInReadonlyModeDueToErrorThreshold)
|
||||||
}
|
}
|
||||||
|
|
||||||
// reportShardErrorBackground increases shard error counter and logs an error.
|
// reportShardErrorBackground increases shard error counter and logs an error.
|
||||||
|
@ -177,6 +181,7 @@ func (e *StorageEngine) reportShardErrorWithFlags(
|
||||||
req := setModeRequest{
|
req := setModeRequest{
|
||||||
errorCount: errCount,
|
errorCount: errCount,
|
||||||
sh: sh,
|
sh: sh,
|
||||||
|
isMeta: errors.As(err, new(metaerr.Error)),
|
||||||
}
|
}
|
||||||
|
|
||||||
select {
|
select {
|
||||||
|
|
|
@ -154,7 +154,7 @@ func TestErrorReporting(t *testing.T) {
|
||||||
for i := uint32(0); i < 2; i++ {
|
for i := uint32(0); i < 2; i++ {
|
||||||
_, err = te.ng.Get(context.Background(), GetPrm{addr: object.AddressOf(obj)})
|
_, err = te.ng.Get(context.Background(), GetPrm{addr: object.AddressOf(obj)})
|
||||||
require.Error(t, err)
|
require.Error(t, err)
|
||||||
checkShardState(t, te.ng, te.shards[0].id, errThreshold+i, mode.DegradedReadOnly)
|
checkShardState(t, te.ng, te.shards[0].id, errThreshold+i, mode.ReadOnly)
|
||||||
checkShardState(t, te.ng, te.shards[1].id, 0, mode.ReadWrite)
|
checkShardState(t, te.ng, te.shards[1].id, 0, mode.ReadWrite)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -84,6 +84,7 @@ func (e *StorageEngine) head(ctx context.Context, prm HeadPrm) (HeadRes, error)
|
||||||
shPrm.SetRaw(prm.raw)
|
shPrm.SetRaw(prm.raw)
|
||||||
|
|
||||||
e.iterateOverSortedShards(prm.addr, func(_ int, sh hashedShard) (stop bool) {
|
e.iterateOverSortedShards(prm.addr, func(_ int, sh hashedShard) (stop bool) {
|
||||||
|
shPrm.ShardLooksBad = sh.errorCount.Load() >= e.errorsThreshold
|
||||||
res, err := sh.Head(ctx, shPrm)
|
res, err := sh.Head(ctx, shPrm)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
switch {
|
switch {
|
||||||
|
|
|
@ -13,8 +13,9 @@ import (
|
||||||
|
|
||||||
// HeadPrm groups the parameters of Head operation.
|
// HeadPrm groups the parameters of Head operation.
|
||||||
type HeadPrm struct {
|
type HeadPrm struct {
|
||||||
addr oid.Address
|
addr oid.Address
|
||||||
raw bool
|
raw bool
|
||||||
|
ShardLooksBad bool
|
||||||
}
|
}
|
||||||
|
|
||||||
// HeadRes groups the resulting values of Head operation.
|
// HeadRes groups the resulting values of Head operation.
|
||||||
|
@ -59,7 +60,8 @@ func (s *Shard) Head(ctx context.Context, prm HeadPrm) (HeadRes, error) {
|
||||||
|
|
||||||
var obj *objectSDK.Object
|
var obj *objectSDK.Object
|
||||||
var err error
|
var err error
|
||||||
if s.GetMode().NoMetabase() {
|
mode := s.GetMode()
|
||||||
|
if mode.NoMetabase() || (mode.ReadOnly() && prm.ShardLooksBad) {
|
||||||
var getPrm GetPrm
|
var getPrm GetPrm
|
||||||
getPrm.SetAddress(prm.addr)
|
getPrm.SetAddress(prm.addr)
|
||||||
getPrm.SetIgnoreMeta(true)
|
getPrm.SetIgnoreMeta(true)
|
||||||
|
|
Loading…
Reference in a new issue