forked from TrueCloudLab/frostfs-node
[#1463] blobovnicza: Determine db size via os.Stat
Currently we use `(*bbolt.Bucket).Stats().KeyN` for estimating database size. However, it iterates over all pages in bucket and thus heavily depends on the bucket size. This commit replaces initial size estimation with a single `os.Stat` call. Signed-off-by: Evgenii Stratonikov <evgeniy@nspcc.ru>
This commit is contained in:
parent
021aa97965
commit
d9f0ac8909
2 changed files with 17 additions and 7 deletions
|
@ -2,6 +2,7 @@ package blobovnicza
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
|
|
||||||
"github.com/nspcc-dev/neofs-node/pkg/util"
|
"github.com/nspcc-dev/neofs-node/pkg/util"
|
||||||
|
@ -53,8 +54,6 @@ func (b *Blobovnicza) Init() error {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
var size uint64
|
|
||||||
|
|
||||||
err := b.boltDB.Update(func(tx *bbolt.Tx) error {
|
err := b.boltDB.Update(func(tx *bbolt.Tx) error {
|
||||||
return b.iterateBucketKeys(func(lower, upper uint64, key []byte) (bool, error) {
|
return b.iterateBucketKeys(func(lower, upper uint64, key []byte) (bool, error) {
|
||||||
// create size range bucket
|
// create size range bucket
|
||||||
|
@ -63,18 +62,25 @@ func (b *Blobovnicza) Init() error {
|
||||||
b.log.Debug("creating bucket for size range",
|
b.log.Debug("creating bucket for size range",
|
||||||
zap.String("range", rangeStr))
|
zap.String("range", rangeStr))
|
||||||
|
|
||||||
buck, err := tx.CreateBucketIfNotExists(key)
|
_, err := tx.CreateBucketIfNotExists(key)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return false, fmt.Errorf("(%T) could not create bucket for bounds %s: %w",
|
return false, fmt.Errorf("(%T) could not create bucket for bounds %s: %w",
|
||||||
b, rangeStr, err)
|
b, rangeStr, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
size += uint64(buck.Stats().KeyN) * (upper + lower) / 2
|
|
||||||
return false, nil
|
return false, nil
|
||||||
})
|
})
|
||||||
})
|
})
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
b.filled.Store(size)
|
info, err := os.Stat(b.path)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("can't determine DB size: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
b.filled.Store(uint64(info.Size()))
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -40,7 +40,11 @@ func TestBlobovniczas(t *testing.T) {
|
||||||
|
|
||||||
c := defaultCfg()
|
c := defaultCfg()
|
||||||
|
|
||||||
var width, depth, szLim uint64 = 2, 2, 2 << 10
|
var width, depth uint64 = 2, 2
|
||||||
|
|
||||||
|
// sizeLim must be big enough, to hold at least multiple pages.
|
||||||
|
// 32 KiB is the initial size after all by-size buckets are created.
|
||||||
|
var szLim uint64 = 32*1024 + 1
|
||||||
|
|
||||||
for _, opt := range []Option{
|
for _, opt := range []Option{
|
||||||
WithLogger(l),
|
WithLogger(l),
|
||||||
|
@ -77,7 +81,7 @@ func TestBlobovniczas(t *testing.T) {
|
||||||
|
|
||||||
// save object in blobovnicza
|
// save object in blobovnicza
|
||||||
id, err := b.put(addr, d)
|
id, err := b.put(addr, d)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err, i)
|
||||||
|
|
||||||
// get w/ blobovnicza ID
|
// get w/ blobovnicza ID
|
||||||
var prm GetSmallPrm
|
var prm GetSmallPrm
|
||||||
|
|
Loading…
Reference in a new issue