From e54dc3dc7c9a581328d4766d4d4a4073a5e47f55 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Thu, 12 Oct 2023 15:25:16 +0300 Subject: [PATCH] [#698] blobovnicza: Store counter values Blobovnicza initialization take a long time because of bucket Stat() call. So now blobovnicza stores counters in META bucket. Signed-off-by: Dmitrii Stepanov --- internal/logs/logs.go | 3 + .../blobovnicza/control.go | 24 ++++ .../blobovnicza/delete.go | 17 ++- .../blobovnicza/exists.go | 3 +- pkg/local_object_storage/blobovnicza/get.go | 2 +- .../blobovnicza/iterate.go | 3 +- pkg/local_object_storage/blobovnicza/meta.go | 103 ++++++++++++++++++ pkg/local_object_storage/blobovnicza/put.go | 4 +- 8 files changed, 152 insertions(+), 7 deletions(-) create mode 100644 pkg/local_object_storage/blobovnicza/meta.go diff --git a/internal/logs/logs.go b/internal/logs/logs.go index 17ba33994..e1c2cfdee 100644 --- a/internal/logs/logs.go +++ b/internal/logs/logs.go @@ -548,4 +548,7 @@ const ( BlobovniczatreeCouldNotCheckExistenceInTargetDB = "could not check object existence in target blobovnicza" BlobovniczatreeCouldNotGetObjectFromSourceDB = "could not get object from source blobovnicza" BlobovniczatreeCouldNotPutObjectToTargetDB = "could not put object to target blobovnicza" + BlobovniczaSavingCountersToMeta = "saving counters to blobovnicza's meta..." + BlobovniczaSavingCountersToMetaSuccess = "saving counters to blobovnicza's meta completed successfully" + BlobovniczaSavingCountersToMetaFailed = "saving counters to blobovnicza's meta failed" ) diff --git a/pkg/local_object_storage/blobovnicza/control.go b/pkg/local_object_storage/blobovnicza/control.go index c1d7d49a3..2a85313f1 100644 --- a/pkg/local_object_storage/blobovnicza/control.go +++ b/pkg/local_object_storage/blobovnicza/control.go @@ -104,7 +104,17 @@ func (b *Blobovnicza) Init() error { func (b *Blobovnicza) initializeCounters() error { var size uint64 var items uint64 + var sizeExists bool + var itemsCountExists bool + err := b.boltDB.View(func(tx *bbolt.Tx) error { + size, sizeExists = hasDataSize(tx) + items, itemsCountExists = hasItemsCount(tx) + + if sizeExists && itemsCountExists { + return nil + } + return b.iterateAllDataBuckets(tx, func(lower, upper uint64, b *bbolt.Bucket) (bool, error) { keysN := uint64(b.Stats().KeyN) size += keysN * upper @@ -115,6 +125,20 @@ func (b *Blobovnicza) initializeCounters() error { if err != nil { return fmt.Errorf("can't determine DB size: %w", err) } + if (!sizeExists || !itemsCountExists) && !b.boltOptions.ReadOnly { + b.log.Debug(logs.BlobovniczaSavingCountersToMeta, zap.Uint64("size", size), zap.Uint64("items", items)) + if err := b.boltDB.Update(func(tx *bbolt.Tx) error { + if err := saveDataSize(tx, size); err != nil { + return err + } + return saveItemsCount(tx, items) + }); err != nil { + b.log.Debug(logs.BlobovniczaSavingCountersToMetaFailed, zap.Uint64("size", size), zap.Uint64("items", items)) + return fmt.Errorf("can't save blobovnicza's size and items count: %w", err) + } + b.log.Debug(logs.BlobovniczaSavingCountersToMetaSuccess, zap.Uint64("size", size), zap.Uint64("items", items)) + } + b.dataSize.Store(size) b.itemsCount.Store(items) b.metrics.AddOpenBlobovniczaSize(size) diff --git a/pkg/local_object_storage/blobovnicza/delete.go b/pkg/local_object_storage/blobovnicza/delete.go index 0f92ea3f5..ff3a7a290 100644 --- a/pkg/local_object_storage/blobovnicza/delete.go +++ b/pkg/local_object_storage/blobovnicza/delete.go @@ -51,7 +51,7 @@ func (b *Blobovnicza) Delete(ctx context.Context, prm DeletePrm) (DeleteRes, err var dataSize uint64 err := b.boltDB.Update(func(tx *bbolt.Tx) error { - return b.iterateAllDataBuckets(tx, func(lower, upper uint64, buck *bbolt.Bucket) (bool, error) { + err := b.iterateAllDataBuckets(tx, func(lower, upper uint64, buck *bbolt.Bucket) (bool, error) { objData := buck.Get(addrKey) if objData == nil { // object is not in bucket => continue iterating @@ -63,6 +63,21 @@ func (b *Blobovnicza) Delete(ctx context.Context, prm DeletePrm) (DeleteRes, err found = true return true, buck.Delete(addrKey) }) + if err != nil { + return err + } + if found { + return updateMeta(tx, func(count, size uint64) (uint64, uint64) { + if count > 0 { + count-- + } + if size >= sizeUpperBound { + size -= sizeUpperBound + } + return count, size + }) + } + return nil }) if err == nil && !found { diff --git a/pkg/local_object_storage/blobovnicza/exists.go b/pkg/local_object_storage/blobovnicza/exists.go index 8cafc72ad..f7bc84d4a 100644 --- a/pkg/local_object_storage/blobovnicza/exists.go +++ b/pkg/local_object_storage/blobovnicza/exists.go @@ -1,7 +1,6 @@ package blobovnicza import ( - "bytes" "context" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" @@ -26,7 +25,7 @@ func (b *Blobovnicza) Exists(ctx context.Context, addr oid.Address) (bool, error err := b.boltDB.View(func(tx *bbolt.Tx) error { return tx.ForEach(func(bucketName []byte, buck *bbolt.Bucket) error { - if bytes.Equal(bucketName, incompletedMoveBucketName) { + if isNonDataBucket(bucketName) { return nil } exists = buck.Get(addrKey) != nil diff --git a/pkg/local_object_storage/blobovnicza/get.go b/pkg/local_object_storage/blobovnicza/get.go index 88d8e7e4a..600323f55 100644 --- a/pkg/local_object_storage/blobovnicza/get.go +++ b/pkg/local_object_storage/blobovnicza/get.go @@ -58,7 +58,7 @@ func (b *Blobovnicza) Get(ctx context.Context, prm GetPrm) (GetRes, error) { if err := b.boltDB.View(func(tx *bbolt.Tx) error { return tx.ForEach(func(bucketName []byte, buck *bbolt.Bucket) error { - if bytes.Equal(bucketName, incompletedMoveBucketName) { + if isNonDataBucket(bucketName) { return nil } diff --git a/pkg/local_object_storage/blobovnicza/iterate.go b/pkg/local_object_storage/blobovnicza/iterate.go index e19f8f14b..32b0ccea7 100644 --- a/pkg/local_object_storage/blobovnicza/iterate.go +++ b/pkg/local_object_storage/blobovnicza/iterate.go @@ -1,7 +1,6 @@ package blobovnicza import ( - "bytes" "context" "fmt" "math" @@ -140,7 +139,7 @@ func (b *Blobovnicza) Iterate(ctx context.Context, prm IteratePrm) (IterateRes, if err := b.boltDB.View(func(tx *bbolt.Tx) error { return tx.ForEach(func(bucketName []byte, buck *bbolt.Bucket) error { - if bytes.Equal(bucketName, incompletedMoveBucketName) { + if isNonDataBucket(bucketName) { return nil } return buck.ForEach(func(k, v []byte) error { diff --git a/pkg/local_object_storage/blobovnicza/meta.go b/pkg/local_object_storage/blobovnicza/meta.go new file mode 100644 index 000000000..8b913f8a7 --- /dev/null +++ b/pkg/local_object_storage/blobovnicza/meta.go @@ -0,0 +1,103 @@ +package blobovnicza + +import ( + "bytes" + "encoding/binary" + + "go.etcd.io/bbolt" +) + +const ( + dataSizeAndItemsCountBufLength = 8 +) + +var ( + metaBucketName = []byte("META") + dataSizeKey = []byte("data_size") + itemsCountKey = []byte("items_count") +) + +func isNonDataBucket(bucketName []byte) bool { + return bytes.Equal(bucketName, incompletedMoveBucketName) || bytes.Equal(bucketName, metaBucketName) +} + +func hasDataSize(tx *bbolt.Tx) (uint64, bool) { + b := tx.Bucket(metaBucketName) + if b == nil { + return 0, false + } + v := b.Get(dataSizeKey) + if v == nil { + return 0, false + } + if len(v) != dataSizeAndItemsCountBufLength { + return 0, false + } + return binary.LittleEndian.Uint64(v), true +} + +func hasItemsCount(tx *bbolt.Tx) (uint64, bool) { + b := tx.Bucket(metaBucketName) + if b == nil { + return 0, false + } + v := b.Get(itemsCountKey) + if v == nil { + return 0, false + } + if len(v) != dataSizeAndItemsCountBufLength { + return 0, false + } + return binary.LittleEndian.Uint64(v), true +} + +func saveDataSize(tx *bbolt.Tx, size uint64) error { + b, err := tx.CreateBucketIfNotExists(metaBucketName) + if err != nil { + return err + } + buf := make([]byte, dataSizeAndItemsCountBufLength) + binary.LittleEndian.PutUint64(buf, size) + return b.Put(dataSizeKey, buf) +} + +func saveItemsCount(tx *bbolt.Tx, count uint64) error { + b, err := tx.CreateBucketIfNotExists(metaBucketName) + if err != nil { + return err + } + buf := make([]byte, dataSizeAndItemsCountBufLength) + binary.LittleEndian.PutUint64(buf, count) + return b.Put(itemsCountKey, buf) +} + +func updateMeta(tx *bbolt.Tx, updateValues func(count, size uint64) (uint64, uint64)) error { + b, err := tx.CreateBucketIfNotExists(metaBucketName) + if err != nil { + return err + } + + var count uint64 + var size uint64 + + v := b.Get(itemsCountKey) + if v != nil { + count = binary.LittleEndian.Uint64(v) + } + + v = b.Get(dataSizeKey) + if v != nil { + size = binary.LittleEndian.Uint64(v) + } + + count, size = updateValues(count, size) + + buf := make([]byte, dataSizeAndItemsCountBufLength) + binary.LittleEndian.PutUint64(buf, size) + if err := b.Put(dataSizeKey, buf); err != nil { + return err + } + + binary.LittleEndian.PutUint64(buf, count) + return b.Put(itemsCountKey, buf) +} diff --git a/pkg/local_object_storage/blobovnicza/put.go b/pkg/local_object_storage/blobovnicza/put.go index 859f7dbbf..608739bee 100644 --- a/pkg/local_object_storage/blobovnicza/put.go +++ b/pkg/local_object_storage/blobovnicza/put.go @@ -73,7 +73,9 @@ func (b *Blobovnicza) Put(ctx context.Context, prm PutPrm) (PutRes, error) { return fmt.Errorf("(%T) could not save object in bucket: %w", b, err) } - return nil + return updateMeta(tx, func(count, size uint64) (uint64, uint64) { + return count + 1, size + upperBound + }) }) if err == nil { b.itemAdded(upperBound)