[#674] storage engine: Use per-shard worker pools for PUT operation
Make `StorageEngine` to use non-blocking worker pools with the same (configurable) size for PUT operation. This allows you to switch to using more free shards when overloading others, thereby more evenly distributing the write load. Signed-off-by: Leonard Lyubich <leonard@nspcc.ru>
This commit is contained in:
parent
b08c6b3f30
commit
5b1975d52a
3 changed files with 73 additions and 36 deletions
|
@ -8,6 +8,7 @@ import (
|
|||
"github.com/nspcc-dev/hrw"
|
||||
"github.com/nspcc-dev/neofs-api-go/pkg/object"
|
||||
"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/shard"
|
||||
"github.com/panjf2000/ants/v2"
|
||||
)
|
||||
|
||||
var errShardNotFound = errors.New("shard not found")
|
||||
|
@ -29,11 +30,20 @@ func (e *StorageEngine) AddShard(opts ...shard.Option) (*shard.ID, error) {
|
|||
return nil, fmt.Errorf("could not generate shard ID: %w", err)
|
||||
}
|
||||
|
||||
e.shards[id.String()] = shard.New(append(opts,
|
||||
pool, err := ants.NewPool(int(e.shardPoolSize), ants.WithNonblocking(true))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
strID := id.String()
|
||||
|
||||
e.shards[strID] = shard.New(append(opts,
|
||||
shard.WithID(id),
|
||||
shard.WithExpiredObjectsCallback(e.processExpiredTombstones),
|
||||
)...)
|
||||
|
||||
e.shardPools[strID] = pool
|
||||
|
||||
return id, nil
|
||||
}
|
||||
|
||||
|
|
Loading…
Add table
Add a link
Reference in a new issue