[#1086] engine: Change mode in case of errors async
All checks were successful
DCO action / DCO (pull_request) Successful in 1m12s
Vulncheck / Vulncheck (pull_request) Successful in 3m9s
Build / Build Components (1.21) (pull_request) Successful in 3m50s
Build / Build Components (1.20) (pull_request) Successful in 3m58s
Tests and linters / Staticcheck (pull_request) Successful in 5m24s
Tests and linters / Lint (pull_request) Successful in 6m17s
Tests and linters / gopls check (pull_request) Successful in 6m39s
Tests and linters / Tests (1.20) (pull_request) Successful in 8m25s
Tests and linters / Tests (1.21) (pull_request) Successful in 10m13s
Tests and linters / Tests with -race (pull_request) Successful in 10m28s

Signed-off-by: Dmitrii Stepanov <d.stepanov@yadro.com>
This commit is contained in:
Dmitrii Stepanov 2024-04-10 11:49:31 +03:00
parent 76398c06b0
commit 1afa563ac5
2 changed files with 19 additions and 21 deletions

View file

@ -133,7 +133,7 @@ func (e *StorageEngine) reportShardErrorBackground(id string, msg string, err er
errCount := sh.errorCount.Add(1)
sh.Shard.IncErrorCounter()
e.reportShardErrorWithFlags(sh.Shard, errCount, false, msg, err)
e.reportShardErrorWithFlags(sh.Shard, errCount, msg, err)
}
// reportShardError checks that the amount of errors doesn't exceed the configured threshold.
@ -153,13 +153,12 @@ func (e *StorageEngine) reportShardError(
errCount := sh.errorCount.Add(1)
sh.Shard.IncErrorCounter()
e.reportShardErrorWithFlags(sh.Shard, errCount, true, msg, err, fields...)
e.reportShardErrorWithFlags(sh.Shard, errCount, msg, err, fields...)
}
func (e *StorageEngine) reportShardErrorWithFlags(
sh *shard.Shard,
errCount uint32,
block bool,
msg string,
err error,
fields ...zap.Field,
@ -175,23 +174,19 @@ func (e *StorageEngine) reportShardErrorWithFlags(
return
}
if block {
e.moveToDegraded(sh, errCount)
} else {
req := setModeRequest{
errorCount: errCount,
sh: sh,
}
req := setModeRequest{
errorCount: errCount,
sh: sh,
}
select {
case e.setModeCh <- req:
default:
// For background workers we can have a lot of such errors,
// thus logging is done with DEBUG level.
e.log.Debug(logs.EngineModeChangeIsInProgressIgnoringSetmodeRequest,
zap.Stringer("shard_id", sid),
zap.Uint32("error_count", errCount))
}
select {
case e.setModeCh <- req:
default:
// For background workers we can have a lot of such errors,
// thus logging is done with DEBUG level.
e.log.Debug(logs.EngineModeChangeIsInProgressIgnoringSetmodeRequest,
zap.Stringer("shard_id", sid),
zap.Uint32("error_count", errCount))
}
}

View file

@ -7,6 +7,7 @@ import (
"path/filepath"
"strconv"
"testing"
"time"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor"
@ -229,6 +230,8 @@ func checkShardState(t *testing.T, e *StorageEngine, id *shard.ID, errCount uint
sh := e.shards[id.String()]
e.mtx.RUnlock()
require.Equal(t, errCount, sh.errorCount.Load())
require.Equal(t, mode, sh.GetMode())
require.Eventually(t, func() bool {
return errCount == sh.errorCount.Load() &&
mode == sh.GetMode()
}, 10*time.Second, 10*time.Millisecond, "shard mode doesn't changed to expected state in 10 seconds")
}