Compare commits

...
Sign in to create a new pull request.

4 commits

Author SHA1 Message Date
72f0d1e24e
[#1692] metabase: Remove useless count variable
All checks were successful
DCO action / DCO (pull_request) Successful in 1m6s
Vulncheck / Vulncheck (pull_request) Successful in 1m24s
Build / Build Components (pull_request) Successful in 2m11s
Pre-commit hooks / Pre-commit (pull_request) Successful in 2m11s
Tests and linters / gopls check (pull_request) Successful in 5m26s
Tests and linters / Staticcheck (pull_request) Successful in 5m54s
Tests and linters / Lint (pull_request) Successful in 6m3s
Tests and linters / Run gofumpt (pull_request) Successful in 5m55s
Tests and linters / Tests (pull_request) Successful in 6m46s
Tests and linters / Tests with -race (pull_request) Successful in 6m57s
It is always equal to `len(to)`.

Change-Id: Id7a4c26e9711216b78f96e6b2511efa0773e3471
Signed-off-by: Evgenii Stratonikov <e.stratonikov@yadro.com>
2025-03-20 17:56:08 +03:00
80bd5efe3f
[#1692] metabase: Use bucket cache in ListWithCursor()
No changes in speed, but unified approach:
```
goos: linux
goarch: amd64
pkg: git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase
cpu: 11th Gen Intel(R) Core(TM) i5-1135G7 @ 2.40GHz
                           │    master    │                 new                 │
                           │    sec/op    │    sec/op     vs base               │
ListWithCursor/1_item-8      6.067µ ±  8%   5.731µ ± 10%       ~ (p=0.052 n=10)
ListWithCursor/10_items-8    25.40µ ± 11%   26.12µ ± 13%       ~ (p=0.971 n=10)
ListWithCursor/100_items-8   210.7µ ±  9%   203.2µ ±  6%       ~ (p=0.280 n=10)
geomean                      31.90µ         31.22µ        -2.16%

                           │    master    │                  new                  │
                           │     B/op     │     B/op      vs base                 │
ListWithCursor/1_item-8      3.287Ki ± 0%   3.287Ki ± 0%       ~ (p=1.000 n=10) ¹
ListWithCursor/10_items-8    15.63Ki ± 0%   15.62Ki ± 0%       ~ (p=0.328 n=10)
ListWithCursor/100_items-8   138.1Ki ± 0%   138.1Ki ± 0%       ~ (p=0.340 n=10)
geomean                      19.21Ki        19.21Ki       -0.00%
¹ all samples are equal

                           │   master    │                 new                  │
                           │  allocs/op  │  allocs/op   vs base                 │
ListWithCursor/1_item-8       109.0 ± 0%    109.0 ± 0%       ~ (p=1.000 n=10) ¹
ListWithCursor/10_items-8     380.0 ± 0%    380.0 ± 0%       ~ (p=1.000 n=10) ¹
ListWithCursor/100_items-8   3.082k ± 0%   3.082k ± 0%       ~ (p=1.000 n=10) ¹
geomean                       503.5         503.5       +0.00%
¹ all samples are equal
```

Change-Id: Ic11673427615053656b2a60068a6d4dbd27af2cb
Signed-off-by: Evgenii Stratonikov <e.stratonikov@yadro.com>
2025-03-20 17:56:08 +03:00
1e660c6c61
[#1692] metabase: Do not allocate map in cache unless needed
Change-Id: I8b1015a8c7c3df4153a08fdb788117d9f0d6c333
Signed-off-by: Evgenii Stratonikov <e.stratonikov@yadro.com>
2025-03-20 17:56:08 +03:00
7eb2f6d0e1
[#1692] metabase/test: Fix end of iteration error check
This is not good:
```
BenchmarkListWithCursor/1_item-8                --- FAIL: BenchmarkListWithCursor/1_item-8
    list_test.go:63: error: end of object listing
```

Change-Id: I61b70937ce30fefaf16ebeb0cdb51bdd39096061
Signed-off-by: Evgenii Stratonikov <e.stratonikov@yadro.com>
2025-03-20 17:56:08 +03:00
3 changed files with 20 additions and 19 deletions

View file

@ -14,10 +14,7 @@ type bucketCache struct {
}
func newBucketCache() *bucketCache {
return &bucketCache{
expired: make(map[cid.ID]*bbolt.Bucket),
primary: make(map[cid.ID]*bbolt.Bucket),
}
return &bucketCache{}
}
func getLockedBucket(bc *bucketCache, tx *bbolt.Tx) *bbolt.Bucket {
@ -56,7 +53,7 @@ func getExpiredBucket(bc *bucketCache, tx *bbolt.Tx, cnr cid.ID) *bbolt.Bucket {
bucketName = objectToExpirationEpochBucketName(cnr, bucketName)
return tx.Bucket(bucketName)
}
return getMappedBucket(bc.expired, tx, objectToExpirationEpochBucketName, cnr)
return getMappedBucket(&bc.expired, tx, objectToExpirationEpochBucketName, cnr)
}
func getPrimaryBucket(bc *bucketCache, tx *bbolt.Tx, cnr cid.ID) *bbolt.Bucket {
@ -65,17 +62,21 @@ func getPrimaryBucket(bc *bucketCache, tx *bbolt.Tx, cnr cid.ID) *bbolt.Bucket {
bucketName = primaryBucketName(cnr, bucketName)
return tx.Bucket(bucketName)
}
return getMappedBucket(bc.primary, tx, primaryBucketName, cnr)
return getMappedBucket(&bc.primary, tx, primaryBucketName, cnr)
}
func getMappedBucket(m map[cid.ID]*bbolt.Bucket, tx *bbolt.Tx, nameFunc func(cid.ID, []byte) []byte, cnr cid.ID) *bbolt.Bucket {
value, ok := m[cnr]
func getMappedBucket(m *map[cid.ID]*bbolt.Bucket, tx *bbolt.Tx, nameFunc func(cid.ID, []byte) []byte, cnr cid.ID) *bbolt.Bucket {
value, ok := (*m)[cnr]
if ok {
return value
}
if *m == nil {
*m = make(map[cid.ID]*bbolt.Bucket, 1)
}
bucketName := make([]byte, bucketKeySize)
bucketName = nameFunc(cnr, bucketName)
m[cnr] = getBucket(&value, tx, bucketName)
(*m)[cnr] = getBucket(&value, tx, bucketName)
return value
}

View file

@ -139,8 +139,7 @@ func (db *DB) listWithCursor(tx *bbolt.Tx, result []objectcore.Info, count int,
var containerID cid.ID
var offset []byte
graveyardBkt := tx.Bucket(graveyardBucketName)
garbageBkt := tx.Bucket(garbageBucketName)
bc := newBucketCache()
rawAddr := make([]byte, cidSize, addressKeySize)
@ -169,7 +168,7 @@ loop:
bkt := tx.Bucket(name)
if bkt != nil {
copy(rawAddr, cidRaw)
result, offset, cursor, err = selectNFromBucket(bkt, objType, graveyardBkt, garbageBkt, rawAddr, containerID,
result, offset, cursor, err = selectNFromBucket(bc, bkt, objType, rawAddr, containerID,
result, count, cursor, threshold, currEpoch)
if err != nil {
return nil, nil, err
@ -204,9 +203,10 @@ loop:
// selectNFromBucket similar to selectAllFromBucket but uses cursor to find
// object to start selecting from. Ignores inhumed objects.
func selectNFromBucket(bkt *bbolt.Bucket, // main bucket
func selectNFromBucket(
bc *bucketCache,
bkt *bbolt.Bucket, // main bucket
objType objectSDK.Type, // type of the objects stored in the main bucket
graveyardBkt, garbageBkt *bbolt.Bucket, // cached graveyard buckets
cidRaw []byte, // container ID prefix, optimization
cnt cid.ID, // container ID
to []objectcore.Info, // listing result
@ -219,7 +219,6 @@ func selectNFromBucket(bkt *bbolt.Bucket, // main bucket
cursor = new(Cursor)
}
count := len(to)
c := bkt.Cursor()
k, v := c.First()
@ -231,7 +230,7 @@ func selectNFromBucket(bkt *bbolt.Bucket, // main bucket
}
for ; k != nil; k, v = c.Next() {
if count >= limit {
if len(to) >= limit {
break
}
@ -241,6 +240,8 @@ func selectNFromBucket(bkt *bbolt.Bucket, // main bucket
}
offset = k
graveyardBkt := getGraveyardBucket(bc, bkt.Tx())
garbageBkt := getGarbageBucket(bc, bkt.Tx())
if inGraveyardWithKey(append(cidRaw, k...), graveyardBkt, garbageBkt) > 0 {
continue
}
@ -251,7 +252,7 @@ func selectNFromBucket(bkt *bbolt.Bucket, // main bucket
}
expEpoch, hasExpEpoch := hasExpirationEpoch(&o)
if hasExpEpoch && expEpoch < currEpoch && !objectLocked(bkt.Tx(), cnt, obj) {
if hasExpEpoch && expEpoch < currEpoch && !objectLockedWithCache(bc, bkt.Tx(), cnt, obj) {
continue
}
@ -273,7 +274,6 @@ func selectNFromBucket(bkt *bbolt.Bucket, // main bucket
a.SetContainer(cnt)
a.SetObject(obj)
to = append(to, objectcore.Info{Address: a, Type: objType, IsLinkingObject: isLinkingObj, ECInfo: ecInfo})
count++
}
return to, offset, cursor, nil

View file

@ -59,7 +59,7 @@ func benchmarkListWithCursor(b *testing.B, db *meta.DB, batchSize int) {
for range b.N {
res, err := db.ListWithCursor(context.Background(), prm)
if err != nil {
if errors.Is(err, meta.ErrEndOfListing) {
if !errors.Is(err, meta.ErrEndOfListing) {
b.Fatalf("error: %v", err)
}
prm.SetCursor(nil)