morph: Provide batch size for container listing explicitly #931

Merged
fyrchik merged 1 commit from fyrchik/frostfs-node:fix-iterator into master 2024-01-29 14:00:13 +00:00
2 changed files with 13 additions and 9 deletions

View file

@ -22,13 +22,13 @@ import (
"github.com/nspcc-dev/neo-go/pkg/rpcclient" "github.com/nspcc-dev/neo-go/pkg/rpcclient"
"github.com/nspcc-dev/neo-go/pkg/rpcclient/actor" "github.com/nspcc-dev/neo-go/pkg/rpcclient/actor"
"github.com/nspcc-dev/neo-go/pkg/rpcclient/gas" "github.com/nspcc-dev/neo-go/pkg/rpcclient/gas"
"github.com/nspcc-dev/neo-go/pkg/rpcclient/invoker"
"github.com/nspcc-dev/neo-go/pkg/rpcclient/nep17" "github.com/nspcc-dev/neo-go/pkg/rpcclient/nep17"
"github.com/nspcc-dev/neo-go/pkg/rpcclient/rolemgmt" "github.com/nspcc-dev/neo-go/pkg/rpcclient/rolemgmt"
"github.com/nspcc-dev/neo-go/pkg/rpcclient/unwrap" "github.com/nspcc-dev/neo-go/pkg/rpcclient/unwrap"
"github.com/nspcc-dev/neo-go/pkg/smartcontract" "github.com/nspcc-dev/neo-go/pkg/smartcontract"
"github.com/nspcc-dev/neo-go/pkg/smartcontract/trigger" "github.com/nspcc-dev/neo-go/pkg/smartcontract/trigger"
"github.com/nspcc-dev/neo-go/pkg/util" "github.com/nspcc-dev/neo-go/pkg/util"
"github.com/nspcc-dev/neo-go/pkg/vm"
"github.com/nspcc-dev/neo-go/pkg/vm/stackitem" "github.com/nspcc-dev/neo-go/pkg/vm/stackitem"
"github.com/nspcc-dev/neo-go/pkg/vm/vmstate" "github.com/nspcc-dev/neo-go/pkg/vm/vmstate"
"github.com/nspcc-dev/neo-go/pkg/wallet" "github.com/nspcc-dev/neo-go/pkg/wallet"
@ -207,16 +207,11 @@ func (c *Client) Invoke(contract util.Uint160, fee fixedn.Fixed8, method string,
return vub, nil return vub, nil
} }
// defaultPrefetchBatchSize is the default number of items to prefetch.
// It is dependent on VM limits (2048 items on stack), the default works for simple items.
// For example, to iterate over 2 field structs, the limit should be divided by 3 = 1 (struct itself) + 2 (fields).
const defaultPrefetchBatchSize = vm.MaxStackSize - 16
// TestInvokeIterator invokes contract method returning an iterator and executes cb on each element. // TestInvokeIterator invokes contract method returning an iterator and executes cb on each element.
// If cb returns an error, the session is closed and this error is returned as-is. // If cb returns an error, the session is closed and this error is returned as-is.
// If the remove neo-go node does not support sessions, `unwrap.ErrNoSessionID` is returned. // If the remove neo-go node does not support sessions, `unwrap.ErrNoSessionID` is returned.
// batchSize is the number of items to prefetch: if the number of items in the iterator is less than batchSize, no session will be created. // batchSize is the number of items to prefetch: if the number of items in the iterator is less than batchSize, no session will be created.
// The default batchSize is 2000 (VM is limited by having 2048 items on stack, so if each iterator item is simple, 2000 items won't hit the limit). // The default batchSize is 100, the default limit from neo-go.
func (c *Client) TestInvokeIterator(cb func(stackitem.Item) error, batchSize int, contract util.Uint160, method string, args ...interface{}) error { func (c *Client) TestInvokeIterator(cb func(stackitem.Item) error, batchSize int, contract util.Uint160, method string, args ...interface{}) error {
start := time.Now() start := time.Now()
success := false success := false
@ -225,7 +220,7 @@ func (c *Client) TestInvokeIterator(cb func(stackitem.Item) error, batchSize int
}() }()
if batchSize <= 0 { if batchSize <= 0 {
batchSize = defaultPrefetchBatchSize batchSize = invoker.DefaultIteratorResultItems
} }
c.switchLock.RLock() c.switchLock.RLock()

View file

@ -40,8 +40,17 @@ func (c *Client) ContainersOf(idUser *user.ID) ([]cid.ID, error) {
return nil return nil
} }
// We would like to have batch size as big as possible,
// to reduce the number of round-trips and avoid creating sessions.
// The limit depends on 2 things:
// 1. VM limits: max 2048 items on stack.
// 2. JSON encoded size for the item with type = 128k.
// It turns out, that for container ID the second limit is hit first,
// 512 is big enough value and it is beautiful.
const batchSize = 512
cnrHash := c.client.ContractAddress() cnrHash := c.client.ContractAddress()
err := c.client.Morph().TestInvokeIterator(cb, 0, cnrHash, containersOfMethod, rawID) err := c.client.Morph().TestInvokeIterator(cb, batchSize, cnrHash, containersOfMethod, rawID)
if err != nil { if err != nil {
if errors.Is(err, unwrap.ErrNoSessionID) { if errors.Is(err, unwrap.ErrNoSessionID) {
return c.List(idUser) return c.List(idUser)