diff --git a/pkg/local_object_storage/engine/engine.go b/pkg/local_object_storage/engine/engine.go index 9967b2e2bc..3061d63832 100644 --- a/pkg/local_object_storage/engine/engine.go +++ b/pkg/local_object_storage/engine/engine.go @@ -6,6 +6,7 @@ import ( "sync/atomic" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/metaerr" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" @@ -45,6 +46,7 @@ type shardWrapper struct { type setModeRequest struct { sh *shard.Shard + isMeta bool errorCount uint32 } @@ -70,7 +72,7 @@ func (e *StorageEngine) setModeLoop() { if !ok { inProgress[sid] = struct{}{} go func() { - e.moveToDegraded(r.sh, r.errorCount) + e.moveToDegraded(r.sh, r.errorCount, r.isMeta) mtx.Lock() delete(inProgress, sid) @@ -82,7 +84,7 @@ func (e *StorageEngine) setModeLoop() { } } -func (e *StorageEngine) moveToDegraded(sh *shard.Shard, errCount uint32) { +func (e *StorageEngine) moveToDegraded(sh *shard.Shard, errCount uint32, isMeta bool) { sid := sh.ID() log := e.log.With( zap.Stringer("shard_id", sid), @@ -91,21 +93,23 @@ func (e *StorageEngine) moveToDegraded(sh *shard.Shard, errCount uint32) { e.mtx.RLock() defer e.mtx.RUnlock() - err := sh.SetMode(mode.DegradedReadOnly) - if err != nil { + if isMeta { + err := sh.SetMode(mode.DegradedReadOnly) + if err == nil { + log.Info(logs.EngineShardIsMovedInDegradedModeDueToErrorThreshold) + return + } log.Error(logs.EngineFailedToMoveShardInDegradedreadonlyModeMovingToReadonly, zap.Error(err)) - - err = sh.SetMode(mode.ReadOnly) - if err != nil { - log.Error(logs.EngineFailedToMoveShardInReadonlyMode, - zap.Error(err)) - } else { - log.Info(logs.EngineShardIsMovedInReadonlyModeDueToErrorThreshold) - } - } else { - log.Info(logs.EngineShardIsMovedInDegradedModeDueToErrorThreshold) } + + err := sh.SetMode(mode.ReadOnly) + if err != nil { + log.Error(logs.EngineFailedToMoveShardInReadonlyMode, zap.Error(err)) + return + } + + log.Info(logs.EngineShardIsMovedInReadonlyModeDueToErrorThreshold) } // reportShardErrorBackground increases shard error counter and logs an error. @@ -169,11 +173,13 @@ func (e *StorageEngine) reportShardErrorWithFlags( return } + isMeta := errors.As(err, new(metaerr.Error)) if block { - e.moveToDegraded(sh, errCount) + e.moveToDegraded(sh, errCount, isMeta) } else { req := setModeRequest{ errorCount: errCount, + isMeta: isMeta, sh: sh, } diff --git a/pkg/local_object_storage/engine/error_test.go b/pkg/local_object_storage/engine/error_test.go index dc28d35fac..18bc72d65d 100644 --- a/pkg/local_object_storage/engine/error_test.go +++ b/pkg/local_object_storage/engine/error_test.go @@ -154,7 +154,7 @@ func TestErrorReporting(t *testing.T) { for i := uint32(0); i < 2; i++ { _, err = te.ng.Get(context.Background(), GetPrm{addr: object.AddressOf(obj)}) require.Error(t, err) - checkShardState(t, te.ng, te.shards[0].id, errThreshold+i, mode.DegradedReadOnly) + checkShardState(t, te.ng, te.shards[0].id, errThreshold+i, mode.ReadOnly) checkShardState(t, te.ng, te.shards[1].id, 0, mode.ReadWrite) } @@ -219,7 +219,7 @@ func TestBlobstorFailback(t *testing.T) { require.ErrorAs(t, err, &apistatus.ObjectOutOfRange{}) } - checkShardState(t, te.ng, te.shards[0].id, 1, mode.DegradedReadOnly) + checkShardState(t, te.ng, te.shards[0].id, 2, mode.ReadOnly) checkShardState(t, te.ng, te.shards[1].id, 0, mode.ReadWrite) }