[#996] metabase: Speed up bucket creation
All checks were successful
Vulncheck / Vulncheck (pull_request) Successful in 4m56s
DCO action / DCO (pull_request) Successful in 5m1s
Build / Build Components (1.21) (pull_request) Successful in 6m26s
Build / Build Components (1.20) (pull_request) Successful in 6m41s
Tests and linters / Lint (pull_request) Successful in 8m14s
Tests and linters / Staticcheck (pull_request) Successful in 8m5s
Tests and linters / Tests (1.21) (pull_request) Successful in 10m13s
Tests and linters / Tests (1.20) (pull_request) Successful in 12m50s
Tests and linters / Tests with -race (pull_request) Successful in 13m44s
All checks were successful
Vulncheck / Vulncheck (pull_request) Successful in 4m56s
DCO action / DCO (pull_request) Successful in 5m1s
Build / Build Components (1.21) (pull_request) Successful in 6m26s
Build / Build Components (1.20) (pull_request) Successful in 6m41s
Tests and linters / Lint (pull_request) Successful in 8m14s
Tests and linters / Staticcheck (pull_request) Successful in 8m5s
Tests and linters / Tests (1.21) (pull_request) Successful in 10m13s
Tests and linters / Tests (1.20) (pull_request) Successful in 12m50s
Tests and linters / Tests with -race (pull_request) Successful in 13m44s
Most of the time it exits, e.g. when it is per-container and use on each object PUT. Bbolt implementation first tries to create bucket and then returns it if it exists. Create operation uses cursor and thus is not very lightweight, we can avoid it. ``` goos: linux goarch: amd64 pkg: git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase cpu: 11th Gen Intel(R) Core(TM) i5-1135G7 @ 2.40GHz │ old │ new │ │ sec/op │ sec/op vs base │ Put/parallel-8 174.4µ ± 3% 163.3µ ± 3% -6.39% (p=0.000 n=10) Put/sequential-8 263.3µ ± 2% 259.0µ ± 1% -1.64% (p=0.000 n=10) geomean 214.3µ 205.6µ -4.05% │ old │ new │ │ B/op │ B/op vs base │ Put/parallel-8 275.3Ki ± 3% 281.1Ki ± 4% ~ (p=0.063 n=10) Put/sequential-8 413.0Ki ± 2% 426.6Ki ± 2% +3.29% (p=0.003 n=10) geomean 337.2Ki 346.3Ki +2.70% │ old │ new │ │ allocs/op │ allocs/op vs base │ Put/parallel-8 678.0 ± 1% 524.5 ± 2% -22.64% (p=0.000 n=10) Put/sequential-8 1.329k ± 0% 1.183k ± 0% -10.91% (p=0.000 n=10) geomean 949.1 787.9 -16.98% ``` Signed-off-by: Evgenii Stratonikov <e.stratonikov@yadro.com>
This commit is contained in:
parent
9adcb253be
commit
9db82319aa
2 changed files with 20 additions and 7 deletions
|
@ -346,7 +346,7 @@ func (db *DB) incContainerObjectCounter(tx *bbolt.Tx, cnrID cid.ID, isUserObject
|
||||||
// Does nothing if counters are not empty and force is false. If force is
|
// Does nothing if counters are not empty and force is false. If force is
|
||||||
// true, updates the counters anyway.
|
// true, updates the counters anyway.
|
||||||
func syncCounter(tx *bbolt.Tx, force bool) error {
|
func syncCounter(tx *bbolt.Tx, force bool) error {
|
||||||
shardInfoB, err := tx.CreateBucketIfNotExists(shardInfoBucket)
|
shardInfoB, err := createBucketLikelyExists(tx, shardInfoBucket)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("could not get shard info bucket: %w", err)
|
return fmt.Errorf("could not get shard info bucket: %w", err)
|
||||||
}
|
}
|
||||||
|
@ -359,7 +359,7 @@ func syncCounter(tx *bbolt.Tx, force bool) error {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
containerCounterB, err := tx.CreateBucketIfNotExists(containerCounterBucketName)
|
containerCounterB, err := createBucketLikelyExists(tx, containerCounterBucketName)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("could not get container counter bucket: %w", err)
|
return fmt.Errorf("could not get container counter bucket: %w", err)
|
||||||
}
|
}
|
||||||
|
|
|
@ -356,8 +356,21 @@ func updateFKBTIndexes(tx *bbolt.Tx, obj *objectSDK.Object, f updateIndexItemFun
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
type bucketContainer interface {
|
||||||
|
Bucket([]byte) *bbolt.Bucket
|
||||||
|
CreateBucket([]byte) (*bbolt.Bucket, error)
|
||||||
|
CreateBucketIfNotExists([]byte) (*bbolt.Bucket, error)
|
||||||
|
}
|
||||||
|
|
||||||
|
func createBucketLikelyExists[T bucketContainer](tx T, name []byte) (*bbolt.Bucket, error) {
|
||||||
|
if bkt := tx.Bucket(name); bkt != nil {
|
||||||
|
return bkt, nil
|
||||||
|
}
|
||||||
|
return tx.CreateBucket(name)
|
||||||
|
}
|
||||||
|
|
||||||
func putUniqueIndexItem(tx *bbolt.Tx, item namedBucketItem) error {
|
func putUniqueIndexItem(tx *bbolt.Tx, item namedBucketItem) error {
|
||||||
bkt, err := tx.CreateBucketIfNotExists(item.name)
|
bkt, err := createBucketLikelyExists(tx, item.name)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("can't create index %v: %w", item.name, err)
|
return fmt.Errorf("can't create index %v: %w", item.name, err)
|
||||||
}
|
}
|
||||||
|
@ -366,12 +379,12 @@ func putUniqueIndexItem(tx *bbolt.Tx, item namedBucketItem) error {
|
||||||
}
|
}
|
||||||
|
|
||||||
func putFKBTIndexItem(tx *bbolt.Tx, item namedBucketItem) error {
|
func putFKBTIndexItem(tx *bbolt.Tx, item namedBucketItem) error {
|
||||||
bkt, err := tx.CreateBucketIfNotExists(item.name)
|
bkt, err := createBucketLikelyExists(tx, item.name)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("can't create index %v: %w", item.name, err)
|
return fmt.Errorf("can't create index %v: %w", item.name, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
fkbtRoot, err := bkt.CreateBucketIfNotExists(item.key)
|
fkbtRoot, err := createBucketLikelyExists(bkt, item.key)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("can't create fake bucket tree index %v: %w", item.key, err)
|
return fmt.Errorf("can't create fake bucket tree index %v: %w", item.key, err)
|
||||||
}
|
}
|
||||||
|
@ -380,7 +393,7 @@ func putFKBTIndexItem(tx *bbolt.Tx, item namedBucketItem) error {
|
||||||
}
|
}
|
||||||
|
|
||||||
func putListIndexItem(tx *bbolt.Tx, item namedBucketItem) error {
|
func putListIndexItem(tx *bbolt.Tx, item namedBucketItem) error {
|
||||||
bkt, err := tx.CreateBucketIfNotExists(item.name)
|
bkt, err := createBucketLikelyExists(tx, item.name)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("can't create index %v: %w", item.name, err)
|
return fmt.Errorf("can't create index %v: %w", item.name, err)
|
||||||
}
|
}
|
||||||
|
@ -474,7 +487,7 @@ func getVarUint(data []byte) (uint64, int, error) {
|
||||||
// storage location to another.
|
// storage location to another.
|
||||||
func updateStorageID(tx *bbolt.Tx, addr oid.Address, id []byte) error {
|
func updateStorageID(tx *bbolt.Tx, addr oid.Address, id []byte) error {
|
||||||
key := make([]byte, bucketKeySize)
|
key := make([]byte, bucketKeySize)
|
||||||
bkt, err := tx.CreateBucketIfNotExists(smallBucketName(addr.Container(), key))
|
bkt, err := createBucketLikelyExists(tx, smallBucketName(addr.Container(), key))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
Loading…
Reference in a new issue