From c6af4a3ec80e6d2170c9f19c0235454a00405f4d Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Wed, 13 Sep 2023 11:25:59 +0300 Subject: [PATCH] [#679] engine: Do not increase error counter on meta mismatch It was introduced in 69e1e6ca to help node determine faulty shards. However, the situation is possible in a real-life scenario: 1. Object O is evacuated from shard A to B. 2. Shard A is unmounted because of lower-level errors. 3. We now have object in meta on A and in blobstor on B. Technically we have it in meta on shard B too, but we still got the error if B goes to a degraded mode. Signed-off-by: Evgenii Stratonikov --- internal/logs/logs.go | 1 + pkg/local_object_storage/engine/error_test.go | 2 +- pkg/local_object_storage/engine/get.go | 9 ++++++--- pkg/local_object_storage/engine/range.go | 8 +++++--- 4 files changed, 13 insertions(+), 7 deletions(-) diff --git a/internal/logs/logs.go b/internal/logs/logs.go index 1c5815c6c..c18d191f8 100644 --- a/internal/logs/logs.go +++ b/internal/logs/logs.go @@ -258,6 +258,7 @@ const ( ShardMetaObjectCounterRead = "meta: object counter read" ShardMetaCantReadContainerList = "meta: can't read container list" ShardMetaCantReadContainerSize = "meta: can't read container size" + ShardMetaInfoPresentButObjectNotFound = "meta info was present, but the object is missing" ShardMetabaseFailureSwitchingMode = "metabase failure, switching mode" ShardCantMoveShardToReadonlySwitchMode = "can't move shard to readonly, switch mode" ShardCouldNotUnmarshalObject = "could not unmarshal object" diff --git a/pkg/local_object_storage/engine/error_test.go b/pkg/local_object_storage/engine/error_test.go index 0a48f8188..90356104e 100644 --- a/pkg/local_object_storage/engine/error_test.go +++ b/pkg/local_object_storage/engine/error_test.go @@ -217,7 +217,7 @@ func TestBlobstorFailback(t *testing.T) { require.True(t, shard.IsErrOutOfRange(err)) } - checkShardState(t, te.ng, te.shards[0].id, 2, mode.ReadOnly) + checkShardState(t, te.ng, te.shards[0].id, 0, mode.ReadWrite) checkShardState(t, te.ng, te.shards[1].id, 0, mode.ReadWrite) } diff --git a/pkg/local_object_storage/engine/get.go b/pkg/local_object_storage/engine/get.go index 2f736146c..5d888238e 100644 --- a/pkg/local_object_storage/engine/get.go +++ b/pkg/local_object_storage/engine/get.go @@ -4,6 +4,7 @@ import ( "context" "errors" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" @@ -100,9 +101,11 @@ func (e *StorageEngine) get(ctx context.Context, prm GetPrm) (GetRes, error) { if it.Object == nil { return GetRes{}, it.OutError } - if it.ShardWithMeta.Shard != nil { - e.reportShardError(it.ShardWithMeta, "meta info was present, but object is missing", - it.MetaError, zap.Stringer("address", prm.addr)) + if it.ShardWithMeta.Shard != nil && it.MetaError != nil { + e.log.Warn(logs.ShardMetaInfoPresentButObjectNotFound, + zap.Stringer("shard_id", it.ShardWithMeta.ID()), + zap.String("error", it.MetaError.Error()), + zap.Stringer("address", prm.addr)) } } diff --git a/pkg/local_object_storage/engine/range.go b/pkg/local_object_storage/engine/range.go index b3aaea6f5..fb0873ee2 100644 --- a/pkg/local_object_storage/engine/range.go +++ b/pkg/local_object_storage/engine/range.go @@ -5,6 +5,7 @@ import ( "errors" "strconv" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" @@ -113,9 +114,10 @@ func (e *StorageEngine) getRange(ctx context.Context, prm RngPrm) (RngRes, error if it.Object == nil { return RngRes{}, it.OutError } - if it.ShardWithMeta.Shard != nil { - e.reportShardError(it.ShardWithMeta, "meta info was present, but object is missing", - it.MetaError, + if it.ShardWithMeta.Shard != nil && it.MetaError != nil { + e.log.Warn(logs.ShardMetaInfoPresentButObjectNotFound, + zap.Stringer("shard_id", it.ShardWithMeta.ID()), + zap.String("error", it.MetaError.Error()), zap.Stringer("address", prm.addr)) } }