From 69df0d21c2d5f14a28d45951c32702ceec9da759 Mon Sep 17 00:00:00 2001
From: Evgenii Stratonikov <e.stratonikov@yadro.com>
Date: Thu, 15 Jun 2023 14:50:51 +0300
Subject: [PATCH] [#446] engine: Move to read-only on blobstor errors

Signed-off-by: Evgenii Stratonikov <e.stratonikov@yadro.com>
---
 pkg/local_object_storage/engine/engine.go     | 36 +++++++++++--------
 pkg/local_object_storage/engine/error_test.go |  4 +--
 2 files changed, 23 insertions(+), 17 deletions(-)

diff --git a/pkg/local_object_storage/engine/engine.go b/pkg/local_object_storage/engine/engine.go
index 9967b2e2..3061d638 100644
--- a/pkg/local_object_storage/engine/engine.go
+++ b/pkg/local_object_storage/engine/engine.go
@@ -6,6 +6,7 @@ import (
 	"sync/atomic"
 
 	"git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs"
+	"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/metaerr"
 	"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard"
 	"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode"
 	"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr"
@@ -45,6 +46,7 @@ type shardWrapper struct {
 
 type setModeRequest struct {
 	sh         *shard.Shard
+	isMeta     bool
 	errorCount uint32
 }
 
@@ -70,7 +72,7 @@ func (e *StorageEngine) setModeLoop() {
 			if !ok {
 				inProgress[sid] = struct{}{}
 				go func() {
-					e.moveToDegraded(r.sh, r.errorCount)
+					e.moveToDegraded(r.sh, r.errorCount, r.isMeta)
 
 					mtx.Lock()
 					delete(inProgress, sid)
@@ -82,7 +84,7 @@ func (e *StorageEngine) setModeLoop() {
 	}
 }
 
-func (e *StorageEngine) moveToDegraded(sh *shard.Shard, errCount uint32) {
+func (e *StorageEngine) moveToDegraded(sh *shard.Shard, errCount uint32, isMeta bool) {
 	sid := sh.ID()
 	log := e.log.With(
 		zap.Stringer("shard_id", sid),
@@ -91,21 +93,23 @@ func (e *StorageEngine) moveToDegraded(sh *shard.Shard, errCount uint32) {
 	e.mtx.RLock()
 	defer e.mtx.RUnlock()
 
-	err := sh.SetMode(mode.DegradedReadOnly)
-	if err != nil {
+	if isMeta {
+		err := sh.SetMode(mode.DegradedReadOnly)
+		if err == nil {
+			log.Info(logs.EngineShardIsMovedInDegradedModeDueToErrorThreshold)
+			return
+		}
 		log.Error(logs.EngineFailedToMoveShardInDegradedreadonlyModeMovingToReadonly,
 			zap.Error(err))
-
-		err = sh.SetMode(mode.ReadOnly)
-		if err != nil {
-			log.Error(logs.EngineFailedToMoveShardInReadonlyMode,
-				zap.Error(err))
-		} else {
-			log.Info(logs.EngineShardIsMovedInReadonlyModeDueToErrorThreshold)
-		}
-	} else {
-		log.Info(logs.EngineShardIsMovedInDegradedModeDueToErrorThreshold)
 	}
+
+	err := sh.SetMode(mode.ReadOnly)
+	if err != nil {
+		log.Error(logs.EngineFailedToMoveShardInReadonlyMode, zap.Error(err))
+		return
+	}
+
+	log.Info(logs.EngineShardIsMovedInReadonlyModeDueToErrorThreshold)
 }
 
 // reportShardErrorBackground increases shard error counter and logs an error.
@@ -169,11 +173,13 @@ func (e *StorageEngine) reportShardErrorWithFlags(
 		return
 	}
 
+	isMeta := errors.As(err, new(metaerr.Error))
 	if block {
-		e.moveToDegraded(sh, errCount)
+		e.moveToDegraded(sh, errCount, isMeta)
 	} else {
 		req := setModeRequest{
 			errorCount: errCount,
+			isMeta:     isMeta,
 			sh:         sh,
 		}
 
diff --git a/pkg/local_object_storage/engine/error_test.go b/pkg/local_object_storage/engine/error_test.go
index dc28d35f..18bc72d6 100644
--- a/pkg/local_object_storage/engine/error_test.go
+++ b/pkg/local_object_storage/engine/error_test.go
@@ -154,7 +154,7 @@ func TestErrorReporting(t *testing.T) {
 		for i := uint32(0); i < 2; i++ {
 			_, err = te.ng.Get(context.Background(), GetPrm{addr: object.AddressOf(obj)})
 			require.Error(t, err)
-			checkShardState(t, te.ng, te.shards[0].id, errThreshold+i, mode.DegradedReadOnly)
+			checkShardState(t, te.ng, te.shards[0].id, errThreshold+i, mode.ReadOnly)
 			checkShardState(t, te.ng, te.shards[1].id, 0, mode.ReadWrite)
 		}
 
@@ -219,7 +219,7 @@ func TestBlobstorFailback(t *testing.T) {
 		require.ErrorAs(t, err, &apistatus.ObjectOutOfRange{})
 	}
 
-	checkShardState(t, te.ng, te.shards[0].id, 1, mode.DegradedReadOnly)
+	checkShardState(t, te.ng, te.shards[0].id, 2, mode.ReadOnly)
 	checkShardState(t, te.ng, te.shards[1].id, 0, mode.ReadWrite)
 }