From 175e9c902fc51be12177f77ce5a251c3185e4487 Mon Sep 17 00:00:00 2001
From: Dmitrii Stepanov <d.stepanov@yadro.com>
Date: Wed, 10 Apr 2024 11:49:31 +0300
Subject: [PATCH] [#1086] engine: Change mode in case of errors async

Signed-off-by: Dmitrii Stepanov <d.stepanov@yadro.com>
---
 pkg/local_object_storage/engine/engine.go     | 33 ++++++++-----------
 pkg/local_object_storage/engine/error_test.go |  7 ++--
 2 files changed, 19 insertions(+), 21 deletions(-)

diff --git a/pkg/local_object_storage/engine/engine.go b/pkg/local_object_storage/engine/engine.go
index 558fe92ed..b8ac6cf97 100644
--- a/pkg/local_object_storage/engine/engine.go
+++ b/pkg/local_object_storage/engine/engine.go
@@ -133,7 +133,7 @@ func (e *StorageEngine) reportShardErrorBackground(id string, msg string, err er
 
 	errCount := sh.errorCount.Add(1)
 	sh.Shard.IncErrorCounter()
-	e.reportShardErrorWithFlags(sh.Shard, errCount, false, msg, err)
+	e.reportShardErrorWithFlags(sh.Shard, errCount, msg, err)
 }
 
 // reportShardError checks that the amount of errors doesn't exceed the configured threshold.
@@ -153,13 +153,12 @@ func (e *StorageEngine) reportShardError(
 
 	errCount := sh.errorCount.Add(1)
 	sh.Shard.IncErrorCounter()
-	e.reportShardErrorWithFlags(sh.Shard, errCount, true, msg, err, fields...)
+	e.reportShardErrorWithFlags(sh.Shard, errCount, msg, err, fields...)
 }
 
 func (e *StorageEngine) reportShardErrorWithFlags(
 	sh *shard.Shard,
 	errCount uint32,
-	block bool,
 	msg string,
 	err error,
 	fields ...zap.Field,
@@ -175,23 +174,19 @@ func (e *StorageEngine) reportShardErrorWithFlags(
 		return
 	}
 
-	if block {
-		e.moveToDegraded(sh, errCount)
-	} else {
-		req := setModeRequest{
-			errorCount: errCount,
-			sh:         sh,
-		}
+	req := setModeRequest{
+		errorCount: errCount,
+		sh:         sh,
+	}
 
-		select {
-		case e.setModeCh <- req:
-		default:
-			// For background workers we can have a lot of such errors,
-			// thus logging is done with DEBUG level.
-			e.log.Debug(logs.EngineModeChangeIsInProgressIgnoringSetmodeRequest,
-				zap.Stringer("shard_id", sid),
-				zap.Uint32("error_count", errCount))
-		}
+	select {
+	case e.setModeCh <- req:
+	default:
+		// For background workers we can have a lot of such errors,
+		// thus logging is done with DEBUG level.
+		e.log.Debug(logs.EngineModeChangeIsInProgressIgnoringSetmodeRequest,
+			zap.Stringer("shard_id", sid),
+			zap.Uint32("error_count", errCount))
 	}
 }
 
diff --git a/pkg/local_object_storage/engine/error_test.go b/pkg/local_object_storage/engine/error_test.go
index ec4287bdd..6c44966f2 100644
--- a/pkg/local_object_storage/engine/error_test.go
+++ b/pkg/local_object_storage/engine/error_test.go
@@ -7,6 +7,7 @@ import (
 	"path/filepath"
 	"strconv"
 	"testing"
+	"time"
 
 	"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object"
 	"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor"
@@ -229,6 +230,8 @@ func checkShardState(t *testing.T, e *StorageEngine, id *shard.ID, errCount uint
 	sh := e.shards[id.String()]
 	e.mtx.RUnlock()
 
-	require.Equal(t, errCount, sh.errorCount.Load())
-	require.Equal(t, mode, sh.GetMode())
+	require.Eventually(t, func() bool {
+		return errCount == sh.errorCount.Load() &&
+			mode == sh.GetMode()
+	}, 10*time.Second, 10*time.Millisecond, "shard mode doesn't changed to expected state in 10 seconds")
 }