``` name old time/op new time/op delta ListWithCursor/1_item-8 6.40µs ±13% 6.45µs ±14% ~ (p=0.739 n=10+10) ListWithCursor/10_items-8 30.9µs ±21% 20.9µs ±17% -32.49% (p=0.000 n=10+10) ListWithCursor/100_items-8 274µs ±27% 153µs ±12% -44.09% (p=0.000 n=10+10) name old alloc/op new alloc/op delta ListWithCursor/1_item-8 2.26kB ± 0% 2.31kB ± 0% +2.46% (p=0.000 n=10+10) ListWithCursor/10_items-8 10.8kB ± 0% 6.9kB ± 0% -36.07% (p=0.000 n=8+8) ListWithCursor/100_items-8 96.8kB ± 0% 53.3kB ± 0% -44.98% (p=0.000 n=10+10) name old allocs/op new allocs/op delta ListWithCursor/1_item-8 39.0 ± 0% 40.0 ± 0% +2.56% (p=0.000 n=10+10) ListWithCursor/10_items-8 192 ± 0% 121 ± 0% -36.98% (p=0.000 n=10+10) ListWithCursor/100_items-8 1.72k ± 0% 0.93k ± 0% -45.93% (p=0.000 n=10+10) ``` Signed-off-by: Evgenii Stratonikov <evgeniy@nspcc.ru> name old time/op new time/op delta ListWithCursor/1_item-8 5.23µs ±19% 5.26µs ±15% ~ (p=0.853 n=10+10) ListWithCursor/10_items-8 27.2µs ±15% 18.0µs ±19% -33.80% (p=0.000 n=10+10) ListWithCursor/100_items-8 250µs ±13% 139µs ±15% -44.27% (p=0.000 n=10+10) name old alloc/op new alloc/op delta ListWithCursor/1_item-8 1.99kB ± 0% 2.04kB ± 0% +2.82% (p=0.000 n=8+8) ListWithCursor/10_items-8 10.3kB ± 0% 6.4kB ± 0% -37.83% (p=0.000 n=8+10) ListWithCursor/100_items-8 93.9kB ± 0% 50.4kB ± 0% -46.37% (p=0.000 n=10+10) name old allocs/op new allocs/op delta ListWithCursor/1_item-8 35.0 ± 0% 36.0 ± 0% +2.86% (p=0.000 n=10+10) ListWithCursor/10_items-8 184 ± 0% 113 ± 0% -38.59% (p=0.000 n=10+10) ListWithCursor/100_items-8 1.67k ± 0% 0.88k ± 0% -47.29% (p=0.000 n=10+10) ``` Signed-off-by: Evgenii Stratonikov <evgeniy@nspcc.ru>
232 lines
5.8 KiB
Go
232 lines
5.8 KiB
Go
package meta
|
|
|
|
import (
|
|
"errors"
|
|
"strings"
|
|
|
|
cid "github.com/nspcc-dev/neofs-sdk-go/container/id"
|
|
oid "github.com/nspcc-dev/neofs-sdk-go/object/id"
|
|
"go.etcd.io/bbolt"
|
|
)
|
|
|
|
// ErrEndOfListing is returned from object listing with cursor
|
|
// when storage can't return any more objects after provided
|
|
// cursor. Use nil cursor object to start listing again.
|
|
var ErrEndOfListing = errors.New("end of object listing")
|
|
|
|
// Cursor is a type for continuous object listing.
|
|
type Cursor struct {
|
|
bucketName []byte
|
|
inBucketOffset []byte
|
|
}
|
|
|
|
// ListPrm contains parameters for ListWithCursor operation.
|
|
type ListPrm struct {
|
|
count int
|
|
cursor *Cursor
|
|
}
|
|
|
|
// WithCount sets maximum amount of addresses that ListWithCursor should return.
|
|
func (l *ListPrm) WithCount(count uint32) {
|
|
l.count = int(count)
|
|
}
|
|
|
|
// WithCursor sets cursor for ListWithCursor operation. For initial request
|
|
// ignore this param or use nil value. For consecutive requests, use value
|
|
// from ListRes.
|
|
func (l *ListPrm) WithCursor(cursor *Cursor) {
|
|
l.cursor = cursor
|
|
}
|
|
|
|
// ListRes contains values returned from ListWithCursor operation.
|
|
type ListRes struct {
|
|
addrList []oid.Address
|
|
cursor *Cursor
|
|
}
|
|
|
|
// AddressList returns addresses selected by ListWithCursor operation.
|
|
func (l ListRes) AddressList() []oid.Address {
|
|
return l.addrList
|
|
}
|
|
|
|
// Cursor returns cursor for consecutive listing requests.
|
|
func (l ListRes) Cursor() *Cursor {
|
|
return l.cursor
|
|
}
|
|
|
|
// ListWithCursor lists physical objects available in metabase starting from
|
|
// cursor. Includes objects of all types. Does not include inhumed objects.
|
|
// Use cursor value from response for consecutive requests.
|
|
//
|
|
// Returns ErrEndOfListing if there are no more objects to return or count
|
|
// parameter set to zero.
|
|
func ListWithCursor(db *DB, count uint32, cursor *Cursor) ([]oid.Address, *Cursor, error) {
|
|
var listPrm ListPrm
|
|
listPrm.WithCount(count)
|
|
listPrm.WithCursor(cursor)
|
|
|
|
r, err := db.ListWithCursor(listPrm)
|
|
if err != nil {
|
|
return nil, nil, err
|
|
}
|
|
|
|
return r.AddressList(), r.Cursor(), nil
|
|
}
|
|
|
|
// ListWithCursor lists physical objects available in metabase starting from
|
|
// cursor. Includes objects of all types. Does not include inhumed objects.
|
|
// Use cursor value from response for consecutive requests.
|
|
//
|
|
// Returns ErrEndOfListing if there are no more objects to return or count
|
|
// parameter set to zero.
|
|
func (db *DB) ListWithCursor(prm ListPrm) (res ListRes, err error) {
|
|
result := make([]oid.Address, 0, prm.count)
|
|
|
|
err = db.boltDB.View(func(tx *bbolt.Tx) error {
|
|
res.addrList, res.cursor, err = db.listWithCursor(tx, result, prm.count, prm.cursor)
|
|
return err
|
|
})
|
|
|
|
return res, err
|
|
}
|
|
|
|
func (db *DB) listWithCursor(tx *bbolt.Tx, result []oid.Address, count int, cursor *Cursor) ([]oid.Address, *Cursor, error) {
|
|
threshold := cursor == nil // threshold is a flag to ignore cursor
|
|
var bucketName []byte
|
|
|
|
c := tx.Cursor()
|
|
name, _ := c.First()
|
|
|
|
if !threshold {
|
|
name, _ = c.Seek(cursor.bucketName)
|
|
}
|
|
|
|
var containerID cid.ID
|
|
|
|
loop:
|
|
for ; name != nil; name, _ = c.Next() {
|
|
postfix, ok := parseContainerIDWithPostfix(&containerID, name)
|
|
if !ok {
|
|
continue
|
|
}
|
|
|
|
switch postfix {
|
|
case
|
|
"",
|
|
storageGroupPostfix,
|
|
bucketNameSuffixLockers,
|
|
tombstonePostfix:
|
|
default:
|
|
continue
|
|
}
|
|
|
|
prefix := containerID.EncodeToString() + "/"
|
|
result, cursor = selectNFromBucket(tx, name, prefix, containerID, result, count, cursor, threshold)
|
|
bucketName = name
|
|
if len(result) >= count {
|
|
break loop
|
|
}
|
|
|
|
// set threshold flag after first `selectNFromBucket` invocation
|
|
// first invocation must look for cursor object
|
|
threshold = true
|
|
}
|
|
|
|
if len(result) == 0 {
|
|
return nil, nil, ErrEndOfListing
|
|
}
|
|
|
|
// new slice is much faster but less memory efficient
|
|
// we need to copy, because bucketName exists during bbolt tx
|
|
cursor.bucketName = make([]byte, len(bucketName))
|
|
copy(cursor.bucketName, bucketName)
|
|
|
|
return result, cursor, nil
|
|
}
|
|
|
|
// selectNFromBucket similar to selectAllFromBucket but uses cursor to find
|
|
// object to start selecting from. Ignores inhumed objects.
|
|
func selectNFromBucket(tx *bbolt.Tx,
|
|
name []byte, // bucket name
|
|
prefix string, // container ID prefix, optimization
|
|
cnt cid.ID, // container ID
|
|
to []oid.Address, // listing result
|
|
limit int, // stop listing at `limit` items in result
|
|
cursor *Cursor, // start from cursor object
|
|
threshold bool, // ignore cursor and start immediately
|
|
) ([]oid.Address, *Cursor) {
|
|
bkt := tx.Bucket(name)
|
|
if bkt == nil {
|
|
return to, cursor
|
|
}
|
|
|
|
if cursor == nil {
|
|
cursor = new(Cursor)
|
|
}
|
|
|
|
count := len(to)
|
|
c := bkt.Cursor()
|
|
k, _ := c.First()
|
|
|
|
offset := cursor.inBucketOffset
|
|
|
|
if !threshold {
|
|
c.Seek(offset)
|
|
k, _ = c.Next() // we are looking for objects _after_ the cursor
|
|
}
|
|
|
|
addrRaw := make([]byte, len(prefix)+44)
|
|
copy(addrRaw, prefix)
|
|
|
|
graveyardBkt := tx.Bucket(graveyardBucketName)
|
|
garbageBkt := tx.Bucket(garbageBucketName)
|
|
|
|
for ; k != nil; k, _ = c.Next() {
|
|
if count >= limit {
|
|
break
|
|
}
|
|
|
|
var obj oid.ID
|
|
if err := obj.DecodeString(string(k)); err != nil {
|
|
break
|
|
}
|
|
|
|
offset = k
|
|
addrRaw = append(addrRaw[:len(prefix)], k...)
|
|
if inGraveyardWithKey(addrRaw, graveyardBkt, garbageBkt) > 0 {
|
|
continue
|
|
}
|
|
|
|
var a oid.Address
|
|
a.SetContainer(cnt)
|
|
a.SetObject(obj)
|
|
to = append(to, a)
|
|
count++
|
|
}
|
|
|
|
// new slice is much faster but less memory efficient
|
|
// we need to copy, because offset exists during bbolt tx
|
|
cursor.inBucketOffset = make([]byte, len(offset))
|
|
copy(cursor.inBucketOffset, offset)
|
|
|
|
return to, cursor
|
|
}
|
|
|
|
func parseContainerIDWithPostfix(containerID *cid.ID, name []byte) (string, bool) {
|
|
var (
|
|
containerIDStr = string(name)
|
|
postfix string
|
|
)
|
|
|
|
ind := strings.Index(string(name), invalidBase58String)
|
|
if ind > 0 {
|
|
postfix = containerIDStr[ind:]
|
|
containerIDStr = containerIDStr[:ind]
|
|
}
|
|
|
|
if err := containerID.DecodeString(containerIDStr); err != nil {
|
|
return "", false
|
|
}
|
|
|
|
return postfix, true
|
|
}
|