From ba374c7907e21a9e97855d853ea517b29d57b9ad Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Tue, 11 Jun 2024 17:05:21 +0300 Subject: [PATCH] Reapply "[#446] engine: Move to read-only on blobstor errors" Signed-off-by: Evgenii Stratonikov --- pkg/local_object_storage/engine/engine.go | 33 +++++++++++-------- pkg/local_object_storage/engine/error_test.go | 2 +- 2 files changed, 20 insertions(+), 15 deletions(-) diff --git a/pkg/local_object_storage/engine/engine.go b/pkg/local_object_storage/engine/engine.go index b8ac6cf97..b87d77e6c 100644 --- a/pkg/local_object_storage/engine/engine.go +++ b/pkg/local_object_storage/engine/engine.go @@ -8,6 +8,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/container" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/metaerr" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" @@ -49,6 +50,7 @@ type shardWrapper struct { type setModeRequest struct { sh *shard.Shard + isMeta bool errorCount uint32 } @@ -74,7 +76,7 @@ func (e *StorageEngine) setModeLoop() { if !ok { inProgress[sid] = struct{}{} go func() { - e.moveToDegraded(r.sh, r.errorCount) + e.moveToDegraded(r.sh, r.errorCount, r.isMeta) mtx.Lock() delete(inProgress, sid) @@ -86,7 +88,7 @@ func (e *StorageEngine) setModeLoop() { } } -func (e *StorageEngine) moveToDegraded(sh *shard.Shard, errCount uint32) { +func (e *StorageEngine) moveToDegraded(sh *shard.Shard, errCount uint32, isMeta bool) { sid := sh.ID() log := e.log.With( zap.Stringer("shard_id", sid), @@ -95,21 +97,23 @@ func (e *StorageEngine) moveToDegraded(sh *shard.Shard, errCount uint32) { e.mtx.RLock() defer e.mtx.RUnlock() - err := sh.SetMode(mode.DegradedReadOnly) - if err != nil { + if isMeta { + err := sh.SetMode(mode.DegradedReadOnly) + if err == nil { + log.Info(logs.EngineShardIsMovedInDegradedModeDueToErrorThreshold) + return + } log.Error(logs.EngineFailedToMoveShardInDegradedreadonlyModeMovingToReadonly, zap.Error(err)) - - err = sh.SetMode(mode.ReadOnly) - if err != nil { - log.Error(logs.EngineFailedToMoveShardInReadonlyMode, - zap.Error(err)) - } else { - log.Info(logs.EngineShardIsMovedInReadonlyModeDueToErrorThreshold) - } - } else { - log.Info(logs.EngineShardIsMovedInDegradedModeDueToErrorThreshold) } + + err := sh.SetMode(mode.ReadOnly) + if err != nil { + log.Error(logs.EngineFailedToMoveShardInReadonlyMode, zap.Error(err)) + return + } + + log.Info(logs.EngineShardIsMovedInReadonlyModeDueToErrorThreshold) } // reportShardErrorBackground increases shard error counter and logs an error. @@ -177,6 +181,7 @@ func (e *StorageEngine) reportShardErrorWithFlags( req := setModeRequest{ errorCount: errCount, sh: sh, + isMeta: errors.As(err, new(metaerr.Error)), } select { diff --git a/pkg/local_object_storage/engine/error_test.go b/pkg/local_object_storage/engine/error_test.go index 6c44966f2..535435ceb 100644 --- a/pkg/local_object_storage/engine/error_test.go +++ b/pkg/local_object_storage/engine/error_test.go @@ -154,7 +154,7 @@ func TestErrorReporting(t *testing.T) { for i := uint32(0); i < 2; i++ { _, err = te.ng.Get(context.Background(), GetPrm{addr: object.AddressOf(obj)}) require.Error(t, err) - checkShardState(t, te.ng, te.shards[0].id, errThreshold+i, mode.DegradedReadOnly) + checkShardState(t, te.ng, te.shards[0].id, errThreshold+i, mode.ReadOnly) checkShardState(t, te.ng, te.shards[1].id, 0, mode.ReadWrite) } -- 2.45.2