TrueCloudLab/frostfs-node
20
1
Fork 27
Code Issues 101 Pull requests 4 Projects Releases 1 Activity Actions

[#679] engine: Do not increase error counter on meta mismatch
All checks were successful
DCO action / DCO (pull_request) Successful in 1m43s
Details
Vulncheck / Vulncheck (pull_request) Successful in 2m0s
Details
Tests and linters / Staticcheck (pull_request) Successful in 2m54s
Details
Build / Build Components (1.20) (pull_request) Successful in 2m49s
Details
Build / Build Components (1.21) (pull_request) Successful in 2m44s
Details
Tests and linters / Tests (1.21) (pull_request) Successful in 3m43s
Details
Tests and linters / Lint (pull_request) Successful in 3m58s
Details
Tests and linters / Tests (1.20) (pull_request) Successful in 23m28s
Details
Tests and linters / Tests with -race (pull_request) Successful in 29m0s
Details

Browse source 
It was introduced in 69e1e6ca to help node determine faulty shards.
However, the situation is possible in a real-life scenario:
1. Object O is evacuated from shard A to B.
2. Shard A is unmounted because of lower-level errors.
3. We now have object in meta on A and in blobstor on B. Technically we
   have it in meta on shard B too, but we still got the error if B goes
   to a degraded mode.

Signed-off-by: Evgenii Stratonikov <e.stratonikov@yadro.com>
This commit is contained in:
Evgenii Stratonikov 2023-09-13 11:25:59 +03:00
parent 58239d1b2c
commit c6af4a3ec8
4 changed files with 13 additions and 7 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
internal/logs/logs.go
View file

@ -258,6 +258,7 @@ const (
ShardMetaObjectCounterRead = "meta: object counter read" ShardMetaObjectCounterRead = "meta: object counter read"
ShardMetaCantReadContainerList = "meta: can't read container list" ShardMetaCantReadContainerList = "meta: can't read container list"
ShardMetaCantReadContainerSize = "meta: can't read container size" ShardMetaCantReadContainerSize = "meta: can't read container size"
ShardMetaInfoPresentButObjectNotFound = "meta info was present, but the object is missing"
ShardMetabaseFailureSwitchingMode = "metabase failure, switching mode" ShardMetabaseFailureSwitchingMode = "metabase failure, switching mode"
ShardCantMoveShardToReadonlySwitchMode = "can't move shard to readonly, switch mode" ShardCantMoveShardToReadonlySwitchMode = "can't move shard to readonly, switch mode"
ShardCouldNotUnmarshalObject = "could not unmarshal object" ShardCouldNotUnmarshalObject = "could not unmarshal object"

2
pkg/local_object_storage/engine/error_test.go
View file

@ -217,7 +217,7 @@ func TestBlobstorFailback(t *testing.T) {
require.True(t, shard.IsErrOutOfRange(err)) require.True(t, shard.IsErrOutOfRange(err))
} }
checkShardState(t, te.ng, te.shards[0].id, 2, mode.ReadOnly) checkShardState(t, te.ng, te.shards[0].id, 0, mode.ReadWrite)
checkShardState(t, te.ng, te.shards[1].id, 0, mode.ReadWrite) checkShardState(t, te.ng, te.shards[1].id, 0, mode.ReadWrite)
} }

9
pkg/local_object_storage/engine/get.go
View file

@ -4,6 +4,7 @@ import (
"context" "context"
"errors" "errors"
"git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr"
@ -100,9 +101,11 @@ func (e *StorageEngine) get(ctx context.Context, prm GetPrm) (GetRes, error) {
if it.Object == nil { if it.Object == nil {
return GetRes{}, it.OutError return GetRes{}, it.OutError
} }
if it.ShardWithMeta.Shard != nil { if it.ShardWithMeta.Shard != nil && it.MetaError != nil {
e.reportShardError(it.ShardWithMeta, "meta info was present, but object is missing", e.log.Warn(logs.ShardMetaInfoPresentButObjectNotFound,
it.MetaError, zap.Stringer("address", prm.addr)) zap.Stringer("shard_id", it.ShardWithMeta.ID()),
zap.String("error", it.MetaError.Error()),
zap.Stringer("address", prm.addr))
} }
} }

8
pkg/local_object_storage/engine/range.go
View file

@ -5,6 +5,7 @@ import (
"errors" "errors"
"strconv" "strconv"
"git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr"
@ -113,9 +114,10 @@ func (e *StorageEngine) getRange(ctx context.Context, prm RngPrm) (RngRes, error
if it.Object == nil { if it.Object == nil {
return RngRes{}, it.OutError return RngRes{}, it.OutError
} }
if it.ShardWithMeta.Shard != nil { if it.ShardWithMeta.Shard != nil && it.MetaError != nil {
e.reportShardError(it.ShardWithMeta, "meta info was present, but object is missing", e.log.Warn(logs.ShardMetaInfoPresentButObjectNotFound,
it.MetaError, zap.Stringer("shard_id", it.ShardWithMeta.ID()),
zap.String("error", it.MetaError.Error()),
zap.Stringer("address", prm.addr)) zap.Stringer("address", prm.addr))
} }
} }