[#799] engine: Skip put when object removed from shard

Signed-off-by: Anton Nikiforov <an.nikiforov@yadro.com>
This commit is contained in:
Anton Nikiforov 2023-11-21 15:06:54 +03:00 committed by Evgenii Stratonikov
parent 6534252c22
commit d0eadf7ea2
2 changed files with 59 additions and 35 deletions

View file

@ -406,9 +406,8 @@ func (e *StorageEngine) tryEvacuateObjectLocal(ctx context.Context, addr oid.Add
if _, ok := shardsToEvacuate[shards[j].ID().String()]; ok { if _, ok := shardsToEvacuate[shards[j].ID().String()]; ok {
continue continue
} }
putDone, exists := e.putToShard(ctx, shards[j].hashedShard, j, shards[j].pool, addr, object) switch e.putToShard(ctx, shards[j].hashedShard, j, shards[j].pool, addr, object).status {
if putDone || exists { case putToShardSuccess:
if putDone {
res.evacuated.Add(1) res.evacuated.Add(1)
e.log.Debug(logs.EngineObjectIsMovedToAnotherShard, e.log.Debug(logs.EngineObjectIsMovedToAnotherShard,
zap.Stringer("from", sh.ID()), zap.Stringer("from", sh.ID()),
@ -416,11 +415,12 @@ func (e *StorageEngine) tryEvacuateObjectLocal(ctx context.Context, addr oid.Add
zap.Stringer("addr", addr), zap.Stringer("addr", addr),
evacuationOperationLogField, evacuationOperationLogField,
zap.String("trace_id", tracingPkg.GetTraceID(ctx))) zap.String("trace_id", tracingPkg.GetTraceID(ctx)))
}
if exists {
res.skipped.Add(1)
}
return true, nil return true, nil
case putToShardExists, putToShardRemoved:
res.skipped.Add(1)
return true, nil
default:
continue
} }
} }

View file

@ -12,6 +12,7 @@ import (
tracingPkg "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/tracing" tracingPkg "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/tracing"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util"
"git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client"
objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
"go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/attribute"
@ -26,6 +27,20 @@ type PutPrm struct {
var errPutShard = errors.New("could not put object to any shard") var errPutShard = errors.New("could not put object to any shard")
type putToShardStatus byte
const (
putToShardUnknown putToShardStatus = iota
putToShardSuccess
putToShardExists
putToShardRemoved
)
type putToShardRes struct {
status putToShardStatus
err error
}
// WithObject is a Put option to set object to save. // WithObject is a Put option to set object to save.
// //
// Option is required. // Option is required.
@ -68,8 +83,7 @@ func (e *StorageEngine) put(ctx context.Context, prm PutPrm) error {
return err return err
} }
finished := false var shRes putToShardRes
e.iterateOverSortedShards(addr, func(ind int, sh hashedShard) (stop bool) { e.iterateOverSortedShards(addr, func(ind int, sh hashedShard) (stop bool) {
e.mtx.RLock() e.mtx.RLock()
pool, ok := e.shardPools[sh.ID().String()] pool, ok := e.shardPools[sh.ID().String()]
@ -78,25 +92,26 @@ func (e *StorageEngine) put(ctx context.Context, prm PutPrm) error {
// Shard was concurrently removed, skip. // Shard was concurrently removed, skip.
return false return false
} }
shRes = e.putToShard(ctx, sh, ind, pool, addr, prm.obj)
putDone, exists := e.putToShard(ctx, sh, ind, pool, addr, prm.obj) return shRes.status != putToShardUnknown
finished = putDone || exists
return finished
}) })
switch shRes.status {
if !finished { case putToShardUnknown:
err = errPutShard return errPutShard
case putToShardRemoved:
return shRes.err
case putToShardExists, putToShardSuccess:
return nil
default:
return errPutShard
} }
return err
} }
// putToShard puts object to sh. // putToShard puts object to sh.
// First return value is true iff put has been successfully done. // Return putToShardStatus and error if it is necessary to propagate an error upper.
// Second return value is true iff object already exists. func (e *StorageEngine) putToShard(ctx context.Context, sh hashedShard, ind int, pool util.WorkerPool,
func (e *StorageEngine) putToShard(ctx context.Context, sh hashedShard, ind int, pool util.WorkerPool, addr oid.Address, obj *objectSDK.Object) (bool, bool) { addr oid.Address, obj *objectSDK.Object,
var putSuccess, alreadyExists bool ) (res putToShardRes) {
exitCh := make(chan struct{}) exitCh := make(chan struct{})
if err := pool.Submit(func() { if err := pool.Submit(func() {
@ -110,14 +125,13 @@ func (e *StorageEngine) putToShard(ctx context.Context, sh hashedShard, ind int,
if shard.IsErrObjectExpired(err) { if shard.IsErrObjectExpired(err) {
// object is already found but // object is already found but
// expired => do nothing with it // expired => do nothing with it
alreadyExists = true res.status = putToShardExists
} }
return // this is not ErrAlreadyRemoved error so we can go to the next shard return // this is not ErrAlreadyRemoved error so we can go to the next shard
} }
alreadyExists = exists.Exists() if exists.Exists() {
if alreadyExists {
if ind != 0 { if ind != 0 {
var toMoveItPrm shard.ToMoveItPrm var toMoveItPrm shard.ToMoveItPrm
toMoveItPrm.SetAddress(addr) toMoveItPrm.SetAddress(addr)
@ -132,6 +146,7 @@ func (e *StorageEngine) putToShard(ctx context.Context, sh hashedShard, ind int,
} }
} }
res.status = putToShardExists
return return
} }
@ -148,12 +163,21 @@ func (e *StorageEngine) putToShard(ctx context.Context, sh hashedShard, ind int,
zap.String("trace_id", tracingPkg.GetTraceID(ctx))) zap.String("trace_id", tracingPkg.GetTraceID(ctx)))
return return
} }
if client.IsErrObjectAlreadyRemoved(err) {
e.log.Warn(logs.EngineCouldNotPutObjectToShard,
zap.Stringer("shard_id", sh.ID()),
zap.String("error", err.Error()),
zap.String("trace_id", tracingPkg.GetTraceID(ctx)))
res.status = putToShardRemoved
res.err = err
return
}
e.reportShardError(sh, "could not put object to shard", err) e.reportShardError(sh, "could not put object to shard", err)
return return
} }
putSuccess = true res.status = putToShardSuccess
}); err != nil { }); err != nil {
e.log.Warn(logs.EngineCouldNotPutObjectToShard, zap.Error(err)) e.log.Warn(logs.EngineCouldNotPutObjectToShard, zap.Error(err))
close(exitCh) close(exitCh)
@ -161,7 +185,7 @@ func (e *StorageEngine) putToShard(ctx context.Context, sh hashedShard, ind int,
<-exitCh <-exitCh
return putSuccess, alreadyExists return
} }
// Put writes provided object to local storage. // Put writes provided object to local storage.