potyarkin/frostfs-node
1
0
Fork 0
forked from TrueCloudLab/frostfs-node
Code Pull requests Activity

[#674] storage engine: Use per-shard worker pools for PUT operation

Browse source 
Make `StorageEngine` to use non-blocking worker pools with the same
(configurable) size for PUT operation. This allows you to switch to using
more free shards when overloading others, thereby more evenly distributing
the write load.

Signed-off-by: Leonard Lyubich <leonard@nspcc.ru>
This commit is contained in:
Leonard Lyubich 2021-10-08 15:25:45 +03:00 • committed by Alex Vanin
parent b08c6b3f30
commit 5b1975d52a
3 changed files with 73 additions and 36 deletions
Download patch file Download diff file Expand all files Collapse all files

76
pkg/local_object_storage/engine/put.go
View file

@ -49,46 +49,58 @@ func (e *StorageEngine) Put(prm *PutPrm) (*PutRes, error) {
finished := false
e.iterateOverSortedShards(prm.obj.Address(), func(ind int, s *shard.Shard) (stop bool) {
exists, err := s.Exists(existPrm)
if err != nil {
return false // this is not ErrAlreadyRemoved error so we can go to the next shard
}
e.mtx.RLock()
pool := e.shardPools[s.ID().String()]
e.mtx.RUnlock()
if exists.Exists() {
if ind != 0 {
toMoveItPrm := new(shard.ToMoveItPrm)
toMoveItPrm.WithAddress(prm.obj.Address())
exitCh := make(chan struct{})
_, err = s.ToMoveIt(toMoveItPrm)
if err != nil {
e.log.Warn("could not mark object for shard relocation",
zap.Stringer("shard", s.ID()),
zap.String("error", err.Error()),
)
_ = pool.Submit(func() {
defer close(exitCh)
exists, err := s.Exists(existPrm)
if err != nil {
return // this is not ErrAlreadyRemoved error so we can go to the next shard
}
if exists.Exists() {
if ind != 0 {
toMoveItPrm := new(shard.ToMoveItPrm)
toMoveItPrm.WithAddress(prm.obj.Address())
_, err = s.ToMoveIt(toMoveItPrm)
if err != nil {
e.log.Warn("could not mark object for shard relocation",
zap.Stringer("shard", s.ID()),
zap.String("error", err.Error()),
)
}
}
finished = true
return
}
putPrm := new(shard.PutPrm)
putPrm.WithObject(prm.obj)
_, err = s.Put(putPrm)
if err != nil {
e.log.Warn("could not put object in shard",
zap.Stringer("shard", s.ID()),
zap.String("error", err.Error()),
)
return
}
finished = true
})
return true
}
<-exitCh
putPrm := new(shard.PutPrm)
putPrm.WithObject(prm.obj)
_, err = s.Put(putPrm)
if err != nil {
e.log.Warn("could not put object in shard",
zap.Stringer("shard", s.ID()),
zap.String("error", err.Error()),
)
return false
}
finished = true
return true
return finished
})
if !finished {