[#1685] metabase: Cache frequently accessed singleton buckets

There are some buckets we access almost always, to check whether an
object is alive. In search we also iterate over lots of objects, and
`tx.Bucket()` shows itself a lot in pprof.
```
goos: linux
goarch: amd64
pkg: git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase
cpu: 11th Gen Intel(R) Core(TM) i5-1135G7 @ 2.40GHz
                          │      1      │                  2                   │
                          │   sec/op    │    sec/op     vs base                │
Select/string_equal-8       4.753m ± 6%   3.969m ± 14%  -16.50% (p=0.000 n=10)
Select/string_not_equal-8   4.247m ± 9%   3.486m ± 11%  -17.93% (p=0.000 n=10)
Select/common_prefix-8      4.163m ± 5%   3.323m ±  5%  -20.18% (p=0.000 n=10)
Select/unknown-8            3.557m ± 3%   3.064m ±  8%  -13.85% (p=0.001 n=10)
geomean                     4.158m        3.445m        -17.15%

                          │      1       │                  2                   │
                          │     B/op     │     B/op      vs base                │
Select/string_equal-8       2.250Mi ± 0%   1.907Mi ± 0%  -15.24% (p=0.000 n=10)
Select/string_not_equal-8   2.250Mi ± 0%   1.907Mi ± 0%  -15.24% (p=0.000 n=10)
Select/common_prefix-8      2.250Mi ± 0%   1.907Mi ± 0%  -15.24% (p=0.000 n=10)
Select/unknown-8            2.243Mi ± 0%   1.900Mi ± 0%  -15.29% (p=0.000 n=10)
geomean                     2.248Mi        1.905Mi       -15.26%

                          │      1      │                  2                  │
                          │  allocs/op  │  allocs/op   vs base                │
Select/string_equal-8       56.02k ± 0%   47.03k ± 0%  -16.05% (p=0.000 n=10)
Select/string_not_equal-8   56.02k ± 0%   47.03k ± 0%  -16.05% (p=0.000 n=10)
Select/common_prefix-8      56.02k ± 0%   47.03k ± 0%  -16.05% (p=0.000 n=10)
Select/unknown-8            55.03k ± 0%   46.04k ± 0%  -16.34% (p=0.000 n=10)
geomean                     55.78k        46.78k       -16.12%
```

Signed-off-by: Evgenii Stratonikov <e.stratonikov@yadro.com>

pkg/local_object_storage/metabase/bucket_cache.go

@@ -0,0 +1,45 @@
+package meta
+
+import (
+	"go.etcd.io/bbolt"
+)
+
+type bucketCache struct {
+	locked    *bbolt.Bucket
+	graveyard *bbolt.Bucket
+	garbage   *bbolt.Bucket
+}
+
+func newBucketCache() *bucketCache {
+	return &bucketCache{}
+}
+
+func getLockedBucket(bc *bucketCache, tx *bbolt.Tx) *bbolt.Bucket {
+	if bc == nil {
+		return tx.Bucket(bucketNameLocked)
+	}
+	return getBucket(&bc.locked, tx, bucketNameLocked)
+}
+
+func getGraveyardBucket(bc *bucketCache, tx *bbolt.Tx) *bbolt.Bucket {
+	if bc == nil {
+		return tx.Bucket(graveyardBucketName)
+	}
+	return getBucket(&bc.graveyard, tx, graveyardBucketName)
+}
+
+func getGarbageBucket(bc *bucketCache, tx *bbolt.Tx) *bbolt.Bucket {
+	if bc == nil {
+		return tx.Bucket(garbageBucketName)
+	}
+	return getBucket(&bc.garbage, tx, garbageBucketName)
+}
+
+func getBucket(cache **bbolt.Bucket, tx *bbolt.Tx, name []byte) *bbolt.Bucket {
+	if *cache != nil {
+		return *cache
+	}
+
+	*cache = tx.Bucket(name)
+	return *cache
+}

pkg/local_object_storage/metabase/exists.go

@@ -153,8 +153,12 @@ func (db *DB) exists(tx *bbolt.Tx, addr oid.Address, ecParent oid.Address, currE
 //   - 2 if object is covered with tombstone;
 //   - 3 if object is expired.
 func objectStatus(tx *bbolt.Tx, addr oid.Address, currEpoch uint64) (uint8, error) {
+	return objectStatusWithCache(nil, tx, addr, currEpoch)
+}
+
+func objectStatusWithCache(bc *bucketCache, tx *bbolt.Tx, addr oid.Address, currEpoch uint64) (uint8, error) {
 	// locked object could not be removed/marked with GC/expired
-	if objectLocked(tx, addr.Container(), addr.Object()) {
+	if objectLockedWithCache(bc, tx, addr.Container(), addr.Object()) {
 		return 0, nil
 	}
 
@@ -167,8 +171,8 @@ func objectStatus(tx *bbolt.Tx, addr oid.Address, currEpoch uint64) (uint8, erro
 		return 3, nil
 	}
 
-	graveyardBkt := tx.Bucket(graveyardBucketName)
-	garbageBkt := tx.Bucket(garbageBucketName)
+	graveyardBkt := getGraveyardBucket(bc, tx)
+	garbageBkt := getGarbageBucket(bc, tx)
 	addrKey := addressKey(addr, make([]byte, addressKeySize))
 	return inGraveyardWithKey(addrKey, graveyardBkt, garbageBkt), nil
 }

pkg/local_object_storage/metabase/lock.go

@@ -163,7 +163,11 @@ func (db *DB) FreeLockedBy(lockers []oid.Address) ([]oid.Address, error) {
 
 // checks if specified object is locked in the specified container.
 func objectLocked(tx *bbolt.Tx, idCnr cid.ID, idObj oid.ID) bool {
-	bucketLocked := tx.Bucket(bucketNameLocked)
+	return objectLockedWithCache(nil, tx, idCnr, idObj)
+}
+
+func objectLockedWithCache(bc *bucketCache, tx *bbolt.Tx, idCnr cid.ID, idObj oid.ID) bool {
+	bucketLocked := getLockedBucket(bc, tx)
 	if bucketLocked != nil {
 		key := make([]byte, cidSize)
 		idCnr.Encode(key)

pkg/local_object_storage/metabase/select.go

@@ -131,6 +131,7 @@ func (db *DB) selectObjects(tx *bbolt.Tx, cnr cid.ID, fs objectSDK.SearchFilters
 
 	res := make([]oid.Address, 0, len(mAddr))
 
+	bc := newBucketCache()
 	for a, ind := range mAddr {
 		if ind != expLen {
 			continue // ignore objects with unmatched fast filters
@@ -145,7 +146,7 @@ func (db *DB) selectObjects(tx *bbolt.Tx, cnr cid.ID, fs objectSDK.SearchFilters
 		var addr oid.Address
 		addr.SetContainer(cnr)
 		addr.SetObject(id)
-		st, err := objectStatus(tx, addr, currEpoch)
+		st, err := objectStatusWithCache(bc, tx, addr, currEpoch)
 		if err != nil {
 			return nil, err
 		}