aarifullin/frostfs-node
1
0
Fork 0
forked from TrueCloudLab/frostfs-node
Code Issues Pull requests Projects Releases Packages Wiki Activity

[#1957] engine: Move shard to read-only if cannot move to degraded

Browse source 
Signed-off-by: Evgenii Stratonikov <evgeniy@morphbits.ru>
This commit is contained in:
Evgenii Stratonikov 2022-10-24 11:31:56 +03:00 committed by fyrchik
parent 7f54462e60
commit 3b939d190c
1 changed files with 17 additions and 3 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

20
pkg/local_object_storage/engine/engine.go
View file

@ -40,9 +40,10 @@ func (e *StorageEngine) reportShardError(
msg string, msg string,
err error, err error,
fields ...zap.Field) { fields ...zap.Field) {
sid := sh.ID()
errCount := sh.errorCount.Inc() errCount := sh.errorCount.Inc()
e.log.Warn(msg, append([]zap.Field{ e.log.Warn(msg, append([]zap.Field{
zap.Stringer("shard_id", sh.ID()), zap.Stringer("shard_id", sid),
zap.Uint32("error count", errCount), zap.Uint32("error count", errCount),
zap.String("error", err.Error()), zap.String("error", err.Error()),
}, fields...)...) }, fields...)...)
@ -53,12 +54,25 @@ func (e *StorageEngine) reportShardError(
err = sh.SetMode(mode.DegradedReadOnly) err = sh.SetMode(mode.DegradedReadOnly)
if err != nil { if err != nil {
e.log.Error("failed to move shard in degraded mode", e.log.Error("failed to move shard in degraded-read-only mode, moving to read-only",
zap.Stringer("shard_id", sid),
zap.Uint32("error count", errCount), zap.Uint32("error count", errCount),
zap.Error(err)) zap.Error(err))
err = sh.SetMode(mode.ReadOnly)
if err != nil {
e.log.Error("failed to move shard in read-only mode",
zap.Stringer("shard_id", sid),
zap.Uint32("error count", errCount),
zap.Error(err))
} else {
e.log.Info("shard is moved in read-only mode due to error threshold",
zap.Stringer("shard_id", sid),
zap.Uint32("error count", errCount))
}
} else { } else {
e.log.Info("shard is moved in degraded mode due to error threshold", e.log.Info("shard is moved in degraded mode due to error threshold",
zap.Stringer("shard_id", sh.ID()), zap.Stringer("shard_id", sid),
zap.Uint32("error count", errCount)) zap.Uint32("error count", errCount))
} }
} }