From 651adf46c6770cb23dfb00a521761faf70bee94f Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Wed, 19 Jan 2022 16:17:36 +0300 Subject: [PATCH] [#1094] blobovnicza: calculate size in `Init` properly If pre-existing blobovnicza is initialized, it's size should be updated even if all buckets are in place. Signed-off-by: Evgenii Stratonikov --- .../blobovnicza/control.go | 35 ++++++++++--------- .../blobovnicza/iterate.go | 4 +-- pkg/local_object_storage/blobovnicza/sizes.go | 23 ------------ 3 files changed, 19 insertions(+), 43 deletions(-) diff --git a/pkg/local_object_storage/blobovnicza/control.go b/pkg/local_object_storage/blobovnicza/control.go index 653808e86..72e2d364b 100644 --- a/pkg/local_object_storage/blobovnicza/control.go +++ b/pkg/local_object_storage/blobovnicza/control.go @@ -1,7 +1,6 @@ package blobovnicza import ( - "errors" "fmt" "path" @@ -49,32 +48,34 @@ func (b *Blobovnicza) Init() error { zap.Uint64("storage size limit", b.fullSizeLimit), ) - return b.boltDB.Update(func(tx *bbolt.Tx) error { + if size := b.filled.Load(); size != 0 { + b.log.Debug("already initialized", zap.Uint64("size", size)) + return nil + } + + var size uint64 + + err := b.boltDB.Update(func(tx *bbolt.Tx) error { return b.iterateBucketKeys(func(lower, upper uint64, key []byte) (bool, error) { // create size range bucket + rangeStr := stringifyBounds(lower, upper) b.log.Debug("creating bucket for size range", - zap.String("range", stringifyBounds(lower, upper)), - ) - - _, err := tx.CreateBucket(key) - if errors.Is(err, bbolt.ErrBucketExists) { - // => "smallest" bucket exists => already initialized => do nothing - // TODO: consider separate bucket structure allocation step - // and state initialization/activation - - b.log.Debug("bucket already exists, initializing state") - - return true, b.syncFullnessCounter(tx) - } + zap.String("range", rangeStr)) + buck, err := tx.CreateBucketIfNotExists(key) if err != nil { - return false, fmt.Errorf("(%T) could not create bucket for bounds [%d:%d]: %w", - b, lower, upper, err) + return false, fmt.Errorf("(%T) could not create bucket for bounds %s: %w", + b, rangeStr, err) } + + size += uint64(buck.Stats().KeyN) * (upper + lower) / 2 return false, nil }) }) + + b.filled.Store(size) + return err } // Close releases all internal database resources. diff --git a/pkg/local_object_storage/blobovnicza/iterate.go b/pkg/local_object_storage/blobovnicza/iterate.go index f7ab27ad3..c1c03e4fd 100644 --- a/pkg/local_object_storage/blobovnicza/iterate.go +++ b/pkg/local_object_storage/blobovnicza/iterate.go @@ -33,9 +33,7 @@ func (b *Blobovnicza) iterateBounds(f func(uint64, uint64) (bool, error)) error for upper := firstBucketBound; upper <= max(objLimitBound, firstBucketBound); upper *= 2 { var lower uint64 - if upper == firstBucketBound { - lower = 0 - } else { + if upper != firstBucketBound { lower = upper/2 + 1 } diff --git a/pkg/local_object_storage/blobovnicza/sizes.go b/pkg/local_object_storage/blobovnicza/sizes.go index a8859ba6d..5b89760c6 100644 --- a/pkg/local_object_storage/blobovnicza/sizes.go +++ b/pkg/local_object_storage/blobovnicza/sizes.go @@ -4,8 +4,6 @@ import ( "encoding/binary" "fmt" "strconv" - - "go.etcd.io/bbolt" ) const firstBucketBound = uint64(32 * 1 << 10) // 32KB @@ -51,24 +49,3 @@ func (b *Blobovnicza) decSize(sz uint64) { func (b *Blobovnicza) full() bool { return b.filled.Load() >= b.fullSizeLimit } - -func (b *Blobovnicza) syncFullnessCounter(tx *bbolt.Tx) error { - sz := uint64(0) - - if err := b.iterateBucketKeys(func(lower, upper uint64, key []byte) (bool, error) { - buck := tx.Bucket(key) - if buck == nil { - return false, fmt.Errorf("bucket not found %s", stringifyBounds(lower, upper)) - } - - sz += uint64(buck.Stats().KeyN) * (upper + lower) / 2 - - return false, nil - }); err != nil { - return err - } - - b.filled.Store(sz) - - return nil -}