From d9f0ac89090dbc0fb9d782a41be7bf853692737b Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Wed, 1 Jun 2022 13:20:51 +0300 Subject: [PATCH] [#1463] blobovnicza: Determine db size via `os.Stat` Currently we use `(*bbolt.Bucket).Stats().KeyN` for estimating database size. However, it iterates over all pages in bucket and thus heavily depends on the bucket size. This commit replaces initial size estimation with a single `os.Stat` call. Signed-off-by: Evgenii Stratonikov --- pkg/local_object_storage/blobovnicza/control.go | 16 +++++++++++----- .../blobstor/blobovnicza_test.go | 8 ++++++-- 2 files changed, 17 insertions(+), 7 deletions(-) diff --git a/pkg/local_object_storage/blobovnicza/control.go b/pkg/local_object_storage/blobovnicza/control.go index 51e872e2..a29b0315 100644 --- a/pkg/local_object_storage/blobovnicza/control.go +++ b/pkg/local_object_storage/blobovnicza/control.go @@ -2,6 +2,7 @@ package blobovnicza import ( "fmt" + "os" "path/filepath" "github.com/nspcc-dev/neofs-node/pkg/util" @@ -53,8 +54,6 @@ func (b *Blobovnicza) Init() error { return nil } - var size uint64 - err := b.boltDB.Update(func(tx *bbolt.Tx) error { return b.iterateBucketKeys(func(lower, upper uint64, key []byte) (bool, error) { // create size range bucket @@ -63,18 +62,25 @@ func (b *Blobovnicza) Init() error { b.log.Debug("creating bucket for size range", zap.String("range", rangeStr)) - buck, err := tx.CreateBucketIfNotExists(key) + _, err := tx.CreateBucketIfNotExists(key) if err != nil { return false, fmt.Errorf("(%T) could not create bucket for bounds %s: %w", b, rangeStr, err) } - size += uint64(buck.Stats().KeyN) * (upper + lower) / 2 return false, nil }) }) + if err != nil { + return err + } - b.filled.Store(size) + info, err := os.Stat(b.path) + if err != nil { + return fmt.Errorf("can't determine DB size: %w", err) + } + + b.filled.Store(uint64(info.Size())) return err } diff --git a/pkg/local_object_storage/blobstor/blobovnicza_test.go b/pkg/local_object_storage/blobstor/blobovnicza_test.go index 70f37cb3..07c0c2d6 100644 --- a/pkg/local_object_storage/blobstor/blobovnicza_test.go +++ b/pkg/local_object_storage/blobstor/blobovnicza_test.go @@ -40,7 +40,11 @@ func TestBlobovniczas(t *testing.T) { c := defaultCfg() - var width, depth, szLim uint64 = 2, 2, 2 << 10 + var width, depth uint64 = 2, 2 + + // sizeLim must be big enough, to hold at least multiple pages. + // 32 KiB is the initial size after all by-size buckets are created. + var szLim uint64 = 32*1024 + 1 for _, opt := range []Option{ WithLogger(l), @@ -77,7 +81,7 @@ func TestBlobovniczas(t *testing.T) { // save object in blobovnicza id, err := b.put(addr, d) - require.NoError(t, err) + require.NoError(t, err, i) // get w/ blobovnicza ID var prm GetSmallPrm