TrueCloudLab/frostfs-node
17
1
Fork 27
Code Issues 89 Pull requests 4 Projects Releases 1 Activity Actions

Reapply "[#446] engine: Move to read-only on blobstor errors" #1172

Merged
fyrchik merged 2 commits from fyrchik/frostfs-node:revert-ro-master into master 2024-06-13 07:35:31 +00:00
Conversation 0 Commits 2 Files changed 4 +26 -18
4 changed files with 26 additions and 18 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

33
pkg/local_object_storage/engine/engine.go
View file

@ -8,6 +8,7 @@ import (
"git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/container"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/metaerr"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr"
@ -49,6 +50,7 @@ type shardWrapper struct {
type setModeRequest struct {
sh *shard.Shard
isMeta bool
errorCount uint32
}
@ -74,7 +76,7 @@ func (e *StorageEngine) setModeLoop() {
if !ok {
inProgress[sid] = struct{}{}
go func() {
e.moveToDegraded(r.sh, r.errorCount)
e.moveToDegraded(r.sh, r.errorCount, r.isMeta)
mtx.Lock()
delete(inProgress, sid)
@ -86,7 +88,7 @@ func (e *StorageEngine) setModeLoop() {
}
}
func (e *StorageEngine) moveToDegraded(sh *shard.Shard, errCount uint32) {
func (e *StorageEngine) moveToDegraded(sh *shard.Shard, errCount uint32, isMeta bool) {
sid := sh.ID()
log := e.log.With(
zap.Stringer("shard_id", sid),
@ -95,21 +97,23 @@ func (e *StorageEngine) moveToDegraded(sh *shard.Shard, errCount uint32) {
e.mtx.RLock()
defer e.mtx.RUnlock()
err := sh.SetMode(mode.DegradedReadOnly)
if err != nil {
if isMeta {
err := sh.SetMode(mode.DegradedReadOnly)
if err == nil {
log.Info(logs.EngineShardIsMovedInDegradedModeDueToErrorThreshold)
return
}
log.Error(logs.EngineFailedToMoveShardInDegradedreadonlyModeMovingToReadonly,
zap.Error(err))
err = sh.SetMode(mode.ReadOnly)
if err != nil {
log.Error(logs.EngineFailedToMoveShardInReadonlyMode,
zap.Error(err))
} else {
log.Info(logs.EngineShardIsMovedInReadonlyModeDueToErrorThreshold)
}
} else {
log.Info(logs.EngineShardIsMovedInDegradedModeDueToErrorThreshold)
}
err := sh.SetMode(mode.ReadOnly)
if err != nil {
log.Error(logs.EngineFailedToMoveShardInReadonlyMode, zap.Error(err))
return
}
log.Info(logs.EngineShardIsMovedInReadonlyModeDueToErrorThreshold)
}
// reportShardErrorBackground increases shard error counter and logs an error.
@ -177,6 +181,7 @@ func (e *StorageEngine) reportShardErrorWithFlags(
req := setModeRequest{
errorCount: errCount,
sh: sh,
isMeta: errors.As(err, new(metaerr.Error)),
}
select {

2
pkg/local_object_storage/engine/error_test.go
View file

@ -154,7 +154,7 @@ func TestErrorReporting(t *testing.T) {
for i := uint32(0); i < 2; i++ {
_, err = te.ng.Get(context.Background(), GetPrm{addr: object.AddressOf(obj)})
require.Error(t, err)
checkShardState(t, te.ng, te.shards[0].id, errThreshold+i, mode.DegradedReadOnly)
checkShardState(t, te.ng, te.shards[0].id, errThreshold+i, mode.ReadOnly)
checkShardState(t, te.ng, te.shards[1].id, 0, mode.ReadWrite)
}

1
pkg/local_object_storage/engine/head.go
View file

@ -84,6 +84,7 @@ func (e *StorageEngine) head(ctx context.Context, prm HeadPrm) (HeadRes, error)
shPrm.SetRaw(prm.raw)
e.iterateOverSortedShards(prm.addr, func(_ int, sh hashedShard) (stop bool) {
shPrm.ShardLooksBad = sh.errorCount.Load() >= e.errorsThreshold
res, err := sh.Head(ctx, shPrm)
if err != nil {
switch {

8
pkg/local_object_storage/shard/head.go
View file

@ -13,8 +13,9 @@ import (
// HeadPrm groups the parameters of Head operation.
type HeadPrm struct {
addr oid.Address
raw bool
addr oid.Address
raw bool
ShardLooksBad bool
}
// HeadRes groups the resulting values of Head operation.
@ -59,7 +60,8 @@ func (s *Shard) Head(ctx context.Context, prm HeadPrm) (HeadRes, error) {
var obj *objectSDK.Object
var err error
if s.GetMode().NoMetabase() {
mode := s.GetMode()
if mode.NoMetabase() || (mode.ReadOnly() && prm.ShardLooksBad) {
var getPrm GetPrm
getPrm.SetAddress(prm.addr)
getPrm.SetIgnoreMeta(true)