TrueCloudLab/frostfs-node
20
1
Fork 27
Code Issues 95 Pull requests 4 Projects Releases 1 Activity Actions

[#661] blobovnicza: Compute size with record size

Browse source 
To get more accurate size of blobovnicza use record
size (lenght of key + lenght of data).

Signed-off-by: Dmitrii Stepanov <d.stepanov@yadro.com>
This commit is contained in:
Dmitrii Stepanov 2023-10-12 21:35:08 +03:00
parent e0cb29cb6e
commit 9de91542fe
5 changed files with 19 additions and 14 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

9
pkg/local_object_storage/blobovnicza/control.go
View file

@ -116,10 +116,11 @@ func (b *Blobovnicza) initializeCounters() error {
} }
return b.iterateAllDataBuckets(tx, func(lower, upper uint64, b *bbolt.Bucket) (bool, error) { return b.iterateAllDataBuckets(tx, func(lower, upper uint64, b *bbolt.Bucket) (bool, error) {
keysN := uint64(b.Stats().KeyN) return false, b.ForEach(func(k, v []byte) error {
size += keysN * upper size += uint64(len(k) + len(v))
items += keysN items++
return false, nil return nil
})
}) })
}) })
if err != nil { if err != nil {

10
pkg/local_object_storage/blobovnicza/delete.go
View file

@ -49,6 +49,7 @@ func (b *Blobovnicza) Delete(ctx context.Context, prm DeletePrm) (DeleteRes, err
var sizeUpperBound uint64 var sizeUpperBound uint64
var sizeLowerBound uint64 var sizeLowerBound uint64
var dataSize uint64 var dataSize uint64
var recordSize uint64
err := b.boltDB.Update(func(tx *bbolt.Tx) error { err := b.boltDB.Update(func(tx *bbolt.Tx) error {
err := b.iterateAllDataBuckets(tx, func(lower, upper uint64, buck *bbolt.Bucket) (bool, error) { err := b.iterateAllDataBuckets(tx, func(lower, upper uint64, buck *bbolt.Bucket) (bool, error) {
@ -60,6 +61,7 @@ func (b *Blobovnicza) Delete(ctx context.Context, prm DeletePrm) (DeleteRes, err
dataSize = uint64(len(objData)) dataSize = uint64(len(objData))
sizeLowerBound = lower sizeLowerBound = lower
sizeUpperBound = upper sizeUpperBound = upper
recordSize = dataSize + uint64(len(addrKey))
found = true found = true
return true, buck.Delete(addrKey) return true, buck.Delete(addrKey)
}) })
@ -71,8 +73,10 @@ func (b *Blobovnicza) Delete(ctx context.Context, prm DeletePrm) (DeleteRes, err
if count > 0 { if count > 0 {
count-- count--
} }
if size >= sizeUpperBound { if size >= recordSize {
size -= sizeUpperBound size -= recordSize
} else {
size = 0
} }
return count, size return count, size
}) })
@ -89,7 +93,7 @@ func (b *Blobovnicza) Delete(ctx context.Context, prm DeletePrm) (DeleteRes, err
zap.String("binary size", stringifyByteSize(dataSize)), zap.String("binary size", stringifyByteSize(dataSize)),
zap.String("range", stringifyBounds(sizeLowerBound, sizeUpperBound)), zap.String("range", stringifyBounds(sizeLowerBound, sizeUpperBound)),
) )
b.itemDeleted(sizeUpperBound) b.itemDeleted(recordSize)
} }
return DeleteRes{}, err return DeleteRes{}, err

7
pkg/local_object_storage/blobovnicza/put.go
View file

@ -57,8 +57,9 @@ func (b *Blobovnicza) Put(ctx context.Context, prm PutPrm) (PutRes, error) {
defer span.End() defer span.End()
sz := uint64(len(prm.objData)) sz := uint64(len(prm.objData))
bucketName, upperBound := bucketForSize(sz) bucketName := bucketForSize(sz)
key := addressKey(prm.addr) key := addressKey(prm.addr)
recordSize := sz + uint64(len(key))
err := b.boltDB.Batch(func(tx *bbolt.Tx) error { err := b.boltDB.Batch(func(tx *bbolt.Tx) error {
buck := tx.Bucket(bucketName) buck := tx.Bucket(bucketName)
@ -75,11 +76,11 @@ func (b *Blobovnicza) Put(ctx context.Context, prm PutPrm) (PutRes, error) {
} }
return updateMeta(tx, func(count, size uint64) (uint64, uint64) { return updateMeta(tx, func(count, size uint64) (uint64, uint64) {
return count + 1, size + upperBound return count + 1, size + recordSize
}) })
}) })
if err == nil { if err == nil {
b.itemAdded(upperBound) b.itemAdded(recordSize)
} }
return PutRes{}, err return PutRes{}, err

5
pkg/local_object_storage/blobovnicza/sizes.go
View file

@ -29,9 +29,8 @@ func bucketKeyFromBounds(upperBound uint64) []byte {
return buf[:ln] return buf[:ln]
} }
func bucketForSize(sz uint64) ([]byte, uint64) { func bucketForSize(sz uint64) []byte {
upperBound := upperPowerOfTwo(sz) return bucketKeyFromBounds(upperPowerOfTwo(sz))
return bucketKeyFromBounds(upperBound), upperBound
} }
func upperPowerOfTwo(v uint64) uint64 { func upperPowerOfTwo(v uint64) uint64 {

2
pkg/local_object_storage/blobovnicza/sizes_test.go
View file

@ -34,7 +34,7 @@ func TestSizes(t *testing.T) {
upperBound: 4 * firstBucketBound, upperBound: 4 * firstBucketBound,
}, },
} { } {
key, _ := bucketForSize(item.sz) key := bucketForSize(item.sz)
require.Equal(t, bucketKeyFromBounds(item.upperBound), key) require.Equal(t, bucketKeyFromBounds(item.upperBound), key)
} }
} }