From 3b939d190c3d8895fd0b7daf1ec91e8d90df27ab Mon Sep 17 00:00:00 2001
From: Evgenii Stratonikov <evgeniy@morphbits.ru>
Date: Mon, 24 Oct 2022 11:31:56 +0300
Subject: [PATCH] [#1957] engine: Move shard to read-only if cannot move to
degraded
Signed-off-by: Evgenii Stratonikov <evgeniy@morphbits.ru>
---
pkg/local_object_storage/engine/engine.go | 20 +++++++++++++++++---
1 file changed, 17 insertions(+), 3 deletions(-)
diff --git a/pkg/local_object_storage/engine/engine.go b/pkg/local_object_storage/engine/engine.go
index 463f2a8f2..33ae46945 100644
--- a/pkg/local_object_storage/engine/engine.go
+++ b/pkg/local_object_storage/engine/engine.go
@@ -40,9 +40,10 @@ func (e *StorageEngine) reportShardError(
msg string,
err error,
fields ...zap.Field) {
+ sid := sh.ID()
errCount := sh.errorCount.Inc()
e.log.Warn(msg, append([]zap.Field{
- zap.Stringer("shard_id", sh.ID()),
+ zap.Stringer("shard_id", sid),
zap.Uint32("error count", errCount),
zap.String("error", err.Error()),
}, fields...)...)
@@ -53,12 +54,25 @@ func (e *StorageEngine) reportShardError(
err = sh.SetMode(mode.DegradedReadOnly)
if err != nil {
- e.log.Error("failed to move shard in degraded mode",
+ e.log.Error("failed to move shard in degraded-read-only mode, moving to read-only",
+ zap.Stringer("shard_id", sid),
zap.Uint32("error count", errCount),
zap.Error(err))
+
+ err = sh.SetMode(mode.ReadOnly)
+ if err != nil {
+ e.log.Error("failed to move shard in read-only mode",
+ zap.Stringer("shard_id", sid),
+ zap.Uint32("error count", errCount),
+ zap.Error(err))
+ } else {
+ e.log.Info("shard is moved in read-only mode due to error threshold",
+ zap.Stringer("shard_id", sid),
+ zap.Uint32("error count", errCount))
+ }
} else {
e.log.Info("shard is moved in degraded mode due to error threshold",
- zap.Stringer("shard_id", sh.ID()),
+ zap.Stringer("shard_id", sid),
zap.Uint32("error count", errCount))
}
}