[#1262] metabase: Do not allocate intermediate slices for indices

```
name              old alloc/op   new alloc/op   delta
Put/parallel-8       123kB ± 4%     119kB ± 3%  -2.72%  (p=0.006 n=10+9)
Put/sequential-8     170kB ± 1%     168kB ± 1%  -1.42%  (p=0.000 n=10+10)

name              old allocs/op  new allocs/op  delta
Put/parallel-8         473 ± 1%       469 ± 0%  -0.87%  (p=0.000 n=10+10)
Put/sequential-8       792 ± 0%       787 ± 0%  -0.58%  (p=0.000 n=10+10)
```

Signed-off-by: Evgenii Stratonikov <evgeniy@nspcc.ru>
This commit is contained in:
Evgenii Stratonikov 2022-03-21 16:59:05 +03:00 committed by Alex Vanin
parent 0e9b6be3fd
commit 43867a3093
2 changed files with 87 additions and 109 deletions

View file

@ -133,36 +133,19 @@ func (db *DB) deleteObject(
obj *objectSDK.Object, obj *objectSDK.Object,
isParent bool, isParent bool,
) error { ) error {
uniqueIndexes, err := delUniqueIndexes(obj, isParent) err := delUniqueIndexes(tx, obj, isParent)
if err != nil { if err != nil {
return fmt.Errorf("can' build unique indexes: %w", err) return fmt.Errorf("can't remove unique indexes")
} }
// delete unique indexes err = updateListIndexes(tx, obj, delListIndexItem)
for i := range uniqueIndexes {
delUniqueIndexItem(tx, uniqueIndexes[i])
}
// build list indexes
listIndexes, err := listIndexes(obj)
if err != nil { if err != nil {
return fmt.Errorf("can' build list indexes: %w", err) return fmt.Errorf("can't remove list indexes: %w", err)
} }
// delete list indexes err = updateFKBTIndexes(tx, obj, delFKBTIndexItem)
for i := range listIndexes {
delListIndexItem(tx, listIndexes[i])
}
// build fake bucket tree indexes
fkbtIndexes, err := fkbtIndexes(obj)
if err != nil { if err != nil {
return fmt.Errorf("can' build fake bucket tree indexes: %w", err) return fmt.Errorf("can't remove fake bucket tree indexes: %w", err)
}
// delete fkbt indexes
for i := range fkbtIndexes {
delFKBTIndexItem(tx, fkbtIndexes[i])
} }
return nil return nil
@ -190,29 +173,30 @@ func delUniqueIndexItem(tx *bbolt.Tx, item namedBucketItem) {
} }
} }
func delFKBTIndexItem(tx *bbolt.Tx, item namedBucketItem) { func delFKBTIndexItem(tx *bbolt.Tx, item namedBucketItem) error {
bkt := tx.Bucket(item.name) bkt := tx.Bucket(item.name)
if bkt == nil { if bkt == nil {
return return nil
} }
fkbtRoot := bkt.Bucket(item.key) fkbtRoot := bkt.Bucket(item.key)
if fkbtRoot == nil { if fkbtRoot == nil {
return return nil
} }
_ = fkbtRoot.Delete(item.val) // ignore error, best effort there _ = fkbtRoot.Delete(item.val) // ignore error, best effort there
return nil
} }
func delListIndexItem(tx *bbolt.Tx, item namedBucketItem) { func delListIndexItem(tx *bbolt.Tx, item namedBucketItem) error {
bkt := tx.Bucket(item.name) bkt := tx.Bucket(item.name)
if bkt == nil { if bkt == nil {
return return nil
} }
lst, err := decodeList(bkt.Get(item.key)) lst, err := decodeList(bkt.Get(item.key))
if err != nil || len(lst) == 0 { if err != nil || len(lst) == 0 {
return return nil
} }
// remove element from the list // remove element from the list
@ -228,25 +212,24 @@ func delListIndexItem(tx *bbolt.Tx, item namedBucketItem) {
if len(newLst) == 0 { if len(newLst) == 0 {
_ = bkt.Delete(item.key) // ignore error, best effort there _ = bkt.Delete(item.key) // ignore error, best effort there
return return nil
} }
// if list is not empty, then update it // if list is not empty, then update it
encodedLst, err := encodeList(lst) encodedLst, err := encodeList(lst)
if err != nil { if err != nil {
return // ignore error, best effort there return nil // ignore error, best effort there
} }
_ = bkt.Put(item.key, encodedLst) // ignore error, best effort there _ = bkt.Put(item.key, encodedLst) // ignore error, best effort there
return nil
} }
func delUniqueIndexes(obj *objectSDK.Object, isParent bool) ([]namedBucketItem, error) { func delUniqueIndexes(tx *bbolt.Tx, obj *objectSDK.Object, isParent bool) error {
addr := object.AddressOf(obj) addr := object.AddressOf(obj)
objKey := objectKey(addr.ObjectID()) objKey := objectKey(addr.ObjectID())
addrKey := addressKey(addr) addrKey := addressKey(addr)
result := make([]namedBucketItem, 0, 5)
// add value to primary unique bucket // add value to primary unique bucket
if !isParent { if !isParent {
var bucketName []byte var bucketName []byte
@ -261,34 +244,32 @@ func delUniqueIndexes(obj *objectSDK.Object, isParent bool) ([]namedBucketItem,
case objectSDK.TypeLock: case objectSDK.TypeLock:
bucketName = bucketNameLockers(*addr.ContainerID()) bucketName = bucketNameLockers(*addr.ContainerID())
default: default:
return nil, ErrUnknownObjectType return ErrUnknownObjectType
} }
result = append(result, namedBucketItem{ delUniqueIndexItem(tx, namedBucketItem{
name: bucketName, name: bucketName,
key: objKey, key: objKey,
}) })
} else { } else {
result = append(result, namedBucketItem{ delUniqueIndexItem(tx, namedBucketItem{
name: parentBucketName(obj.ContainerID()), name: parentBucketName(obj.ContainerID()),
key: objKey, key: objKey,
}) })
} }
result = append(result, delUniqueIndexItem(tx, namedBucketItem{ // remove from small blobovnicza id index
namedBucketItem{ // remove from small blobovnicza id index name: smallBucketName(addr.ContainerID()),
name: smallBucketName(addr.ContainerID()), key: objKey,
key: objKey, })
}, delUniqueIndexItem(tx, namedBucketItem{ // remove from root index
namedBucketItem{ // remove from root index name: rootBucketName(addr.ContainerID()),
name: rootBucketName(addr.ContainerID()), key: objKey,
key: objKey, })
}, delUniqueIndexItem(tx, namedBucketItem{ // remove from ToMoveIt index
namedBucketItem{ // remove from ToMoveIt index name: toMoveItBucketName,
name: toMoveItBucketName, key: addrKey,
key: addrKey, })
},
)
return result, nil return nil
} }

View file

@ -129,46 +129,19 @@ func (db *DB) put(tx *bbolt.Tx, obj *objectSDK.Object, id *blobovnicza.ID, si *o
} }
} }
// build unique indexes err = putUniqueIndexes(tx, obj, si, id)
uniqueIndexes, err := uniqueIndexes(obj, si, id)
if err != nil { if err != nil {
return fmt.Errorf("can' build unique indexes: %w", err) return fmt.Errorf("can't put unique indexes: %w", err)
} }
// put unique indexes err = updateListIndexes(tx, obj, putListIndexItem)
for i := range uniqueIndexes {
err = putUniqueIndexItem(tx, uniqueIndexes[i])
if err != nil {
return err
}
}
// build list indexes
listIndexes, err := listIndexes(obj)
if err != nil { if err != nil {
return fmt.Errorf("can' build list indexes: %w", err) return fmt.Errorf("can't put list indexes: %w", err)
} }
// put list indexes err = updateFKBTIndexes(tx, obj, putFKBTIndexItem)
for i := range listIndexes {
err = putListIndexItem(tx, listIndexes[i])
if err != nil {
return err
}
}
// build fake bucket tree indexes
fkbtIndexes, err := fkbtIndexes(obj)
if err != nil { if err != nil {
return fmt.Errorf("can' build fake bucket tree indexes: %w", err) return fmt.Errorf("can't put fake bucket tree indexes: %w", err)
}
// put fake bucket tree indexes
for i := range fkbtIndexes {
err = putFKBTIndexItem(tx, fkbtIndexes[i])
if err != nil {
return err
}
} }
// update container volume size estimation // update container volume size estimation
@ -187,12 +160,15 @@ func (db *DB) put(tx *bbolt.Tx, obj *objectSDK.Object, id *blobovnicza.ID, si *o
return nil return nil
} }
// builds list of <unique> indexes from the object. func putUniqueIndexes(
func uniqueIndexes(obj *objectSDK.Object, si *objectSDK.SplitInfo, id *blobovnicza.ID) ([]namedBucketItem, error) { tx *bbolt.Tx,
obj *objectSDK.Object,
si *objectSDK.SplitInfo,
id *blobovnicza.ID,
) error {
isParent := si != nil isParent := si != nil
addr := object.AddressOf(obj) addr := object.AddressOf(obj)
objKey := objectKey(addr.ObjectID()) objKey := objectKey(addr.ObjectID())
result := make([]namedBucketItem, 0, 3)
// add value to primary unique bucket // add value to primary unique bucket
if !isParent { if !isParent {
@ -208,27 +184,33 @@ func uniqueIndexes(obj *objectSDK.Object, si *objectSDK.SplitInfo, id *blobovnic
case objectSDK.TypeLock: case objectSDK.TypeLock:
bucketName = bucketNameLockers(*addr.ContainerID()) bucketName = bucketNameLockers(*addr.ContainerID())
default: default:
return nil, ErrUnknownObjectType return ErrUnknownObjectType
} }
rawObject, err := obj.CutPayload().Marshal() rawObject, err := obj.CutPayload().Marshal()
if err != nil { if err != nil {
return nil, fmt.Errorf("can't marshal object header: %w", err) return fmt.Errorf("can't marshal object header: %w", err)
} }
result = append(result, namedBucketItem{ err = putUniqueIndexItem(tx, namedBucketItem{
name: bucketName, name: bucketName,
key: objKey, key: objKey,
val: rawObject, val: rawObject,
}) })
if err != nil {
return err
}
// index blobovniczaID if it is present // index blobovniczaID if it is present
if id != nil { if id != nil {
result = append(result, namedBucketItem{ err = putUniqueIndexItem(tx, namedBucketItem{
name: smallBucketName(addr.ContainerID()), name: smallBucketName(addr.ContainerID()),
key: objKey, key: objKey,
val: *id, val: *id,
}) })
if err != nil {
return err
}
} }
} }
@ -242,79 +224,94 @@ func uniqueIndexes(obj *objectSDK.Object, si *objectSDK.SplitInfo, id *blobovnic
if isParent { if isParent {
splitInfo, err = si.Marshal() splitInfo, err = si.Marshal()
if err != nil { if err != nil {
return nil, fmt.Errorf("can't marshal split info: %w", err) return fmt.Errorf("can't marshal split info: %w", err)
} }
} }
result = append(result, namedBucketItem{ err = putUniqueIndexItem(tx, namedBucketItem{
name: rootBucketName(addr.ContainerID()), name: rootBucketName(addr.ContainerID()),
key: objKey, key: objKey,
val: splitInfo, val: splitInfo,
}) })
if err != nil {
return err
}
} }
return result, nil return nil
} }
// builds list of <list> indexes from the object. type updateIndexItemFunc = func(tx *bbolt.Tx, item namedBucketItem) error
func listIndexes(obj *objectSDK.Object) ([]namedBucketItem, error) {
result := make([]namedBucketItem, 0, 3) func updateListIndexes(tx *bbolt.Tx, obj *objectSDK.Object, f updateIndexItemFunc) error {
addr := object.AddressOf(obj) addr := object.AddressOf(obj)
objKey := objectKey(addr.ObjectID()) objKey := objectKey(addr.ObjectID())
// index payload hashes // index payload hashes
result = append(result, namedBucketItem{ err := f(tx, namedBucketItem{
name: payloadHashBucketName(addr.ContainerID()), name: payloadHashBucketName(addr.ContainerID()),
key: obj.PayloadChecksum().Sum(), key: obj.PayloadChecksum().Sum(),
val: objKey, val: objKey,
}) })
if err != nil {
return err
}
// index parent ids // index parent ids
if obj.ParentID() != nil { if obj.ParentID() != nil {
result = append(result, namedBucketItem{ err := f(tx, namedBucketItem{
name: parentBucketName(addr.ContainerID()), name: parentBucketName(addr.ContainerID()),
key: objectKey(obj.ParentID()), key: objectKey(obj.ParentID()),
val: objKey, val: objKey,
}) })
if err != nil {
return err
}
} }
// index split ids // index split ids
if obj.SplitID() != nil { if obj.SplitID() != nil {
result = append(result, namedBucketItem{ err := f(tx, namedBucketItem{
name: splitBucketName(addr.ContainerID()), name: splitBucketName(addr.ContainerID()),
key: obj.SplitID().ToV2(), key: obj.SplitID().ToV2(),
val: objKey, val: objKey,
}) })
if err != nil {
return err
}
} }
return result, nil return nil
} }
// builds list of <fake bucket tree> indexes from the object. func updateFKBTIndexes(tx *bbolt.Tx, obj *objectSDK.Object, f updateIndexItemFunc) error {
func fkbtIndexes(obj *objectSDK.Object) ([]namedBucketItem, error) {
addr := object.AddressOf(obj) addr := object.AddressOf(obj)
objKey := []byte(addr.ObjectID().String()) objKey := []byte(addr.ObjectID().String())
attrs := obj.Attributes() attrs := obj.Attributes()
result := make([]namedBucketItem, 0, 1+len(attrs))
// owner err := f(tx, namedBucketItem{
result = append(result, namedBucketItem{
name: ownerBucketName(addr.ContainerID()), name: ownerBucketName(addr.ContainerID()),
key: []byte(obj.OwnerID().String()), key: []byte(obj.OwnerID().String()),
val: objKey, val: objKey,
}) })
if err != nil {
return err
}
// user specified attributes // user specified attributes
for i := range attrs { for i := range attrs {
result = append(result, namedBucketItem{ err := f(tx, namedBucketItem{
name: attributeBucketName(addr.ContainerID(), attrs[i].Key()), name: attributeBucketName(addr.ContainerID(), attrs[i].Key()),
key: []byte(attrs[i].Value()), key: []byte(attrs[i].Value()),
val: objKey, val: objKey,
}) })
if err != nil {
return err
}
} }
return result, nil return nil
} }
func putUniqueIndexItem(tx *bbolt.Tx, item namedBucketItem) error { func putUniqueIndexItem(tx *bbolt.Tx, item namedBucketItem) error {