realloc/frostfs-node
1
0
Fork 0
forked from TrueCloudLab/frostfs-node
Code Releases Packages Activity

[#1577] container: Reduce iterations through container list

Browse source 
* Separated iteration through container ids from `ContainersOf()`
  so that it could be reused.
* When listing containers we used to iterate through the
  the whole list of containers twice: first when reading from
  a contract, then when sending them. Now we can send batches
  of containers when reading from the contract.

Signed-off-by: Ekaterina Lebedeva <ekaterina.lebedeva@yadro.com>
This commit is contained in:
Ekaterina Lebedeva 2024-12-25 14:25:29 +03:00
parent 6fe34d266a
commit 242f0095d0
4 changed files with 94 additions and 53 deletions
Download patch file Download diff file Expand all files Collapse all files

50
pkg/morph/client/container/containers_of.go
View file

@ -2,9 +2,7 @@ package container
import (
"errors"
"fmt"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client"
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user"
"github.com/nspcc-dev/neo-go/pkg/rpcclient/unwrap"
@ -16,27 +14,37 @@ import (
//
// If remote RPC does not support neo-go session API, fallback to List() method.
func (c *Client) ContainersOf(idUser *user.ID) ([]cid.ID, error) {
var rawID []byte
var cidList []cid.ID
var err error
cb := func(id cid.ID) error {
cidList = append(cidList, id)
return nil
}
if err = c.IterateContainersOf(idUser, cb); err != nil {
return nil, err
}
return cidList, nil
}
// iterateContainers iterates over a list of container identifiers
// belonging to the specified user of FrostFS system and executes
// `cb` on each element. If idUser is nil, calls it on the list of all containers.
func (c *Client) IterateContainersOf(idUser *user.ID, cb func(item cid.ID) error) error {
var rawID []byte
if idUser != nil {
rawID = idUser.WalletBytes()
}
var cidList []cid.ID
cb := func(item stackitem.Item) error {
rawID, err := client.BytesFromStackItem(item)
cnrHash := c.client.ContractAddress()
itemCb := func(item stackitem.Item) error {
id, err := getCIDfromStackItem(item)
if err != nil {
return fmt.Errorf("get byte array from stack item (%s): %w", containersOfMethod, err)
return err
}
var id cid.ID
err = id.Decode(rawID)
if err != nil {
return fmt.Errorf("decode container ID: %w", err)
if err = cb(id); err != nil {
return err
}
cidList = append(cidList, id)
return nil
}
@ -49,14 +57,10 @@ func (c *Client) ContainersOf(idUser *user.ID) ([]cid.ID, error) {
// 512 is big enough value and it is beautiful.
const batchSize = 512
cnrHash := c.client.ContractAddress()
err := c.client.Morph().TestInvokeIterator(cb, batchSize, cnrHash, containersOfMethod, rawID)
if err != nil {
if errors.Is(err, unwrap.ErrNoSessionID) {
return c.list(idUser)
}
return nil, err
err := c.client.Morph().TestInvokeIterator(itemCb, batchSize, cnrHash, containersOfMethod, rawID)
if err != nil && errors.Is(err, unwrap.ErrNoSessionID) {
return c.iterate(idUser, cb)
}
return cidList, nil
return err
}

44
pkg/morph/client/container/list.go
View file

@ -6,15 +6,16 @@ import (
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client"
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user"
"github.com/nspcc-dev/neo-go/pkg/vm/stackitem"
)
// list returns a list of container identifiers belonging
// iterate iterates through a list of container identifiers belonging
// to the specified user of FrostFS system. The list is composed
// through Container contract call.
//
// Returns the identifiers of all FrostFS containers if pointer
// Iterates through the identifiers of all FrostFS containers if pointer
// to user identifier is nil.
func (c *Client) list(idUser *user.ID) ([]cid.ID, error) {
func (c *Client) iterate(idUser *user.ID, cb func(cid.ID) error) error {
var rawID []byte
if idUser != nil {
@ -27,32 +28,41 @@ func (c *Client) list(idUser *user.ID) ([]cid.ID, error) {
res, err := c.client.TestInvoke(prm)
if err != nil {
return nil, fmt.Errorf("test invoke (%s): %w", listMethod, err)
return fmt.Errorf("test invoke (%s): %w", listMethod, err)
} else if ln := len(res); ln != 1 {
return nil, fmt.Errorf("unexpected stack item count (%s): %d", listMethod, ln)
return fmt.Errorf("unexpected stack item count (%s): %d", listMethod, ln)
}
res, err = client.ArrayFromStackItem(res[0])
if err != nil {
return nil, fmt.Errorf("get stack item array from stack item (%s): %w", listMethod, err)
return fmt.Errorf("get stack item array from stack item (%s): %w", listMethod, err)
}
cidList := make([]cid.ID, 0, len(res))
for i := range res {
rawID, err := client.BytesFromStackItem(res[i])
id, err := getCIDfromStackItem(res[i])
if err != nil {
return nil, fmt.Errorf("get byte array from stack item (%s): %w", listMethod, err)
return err
}
var id cid.ID
err = id.Decode(rawID)
if err != nil {
return nil, fmt.Errorf("decode container ID: %w", err)
if err = cb(id); err != nil {
return err
}
cidList = append(cidList, id)
}
return cidList, nil
return nil
}
func getCIDfromStackItem(item stackitem.Item) (cid.ID, error) {
rawID, err := client.BytesFromStackItem(item)
if err != nil {
return cid.ID{}, fmt.Errorf("get byte array from stack item (%s): %w", listMethod, err)
}
var id cid.ID
err = id.Decode(rawID)
if err != nil {
return cid.ID{}, fmt.Errorf("decode container ID: %w", err)
}
return id, nil
}