Merge pull request #2414 from nspcc-dev/eliminate-getstorageitems

Eliminate GetStorageItems

internal/fakechain/fakechain.go

@@ -325,11 +325,6 @@ func (chain *FakeChain) GetTestVM(t trigger.Type, tx *transaction.Transaction, b
 	panic("TODO")
 }
 
-// GetStorageItems implements Blockchainer interface.
-func (chain *FakeChain) GetStorageItems(id int32) ([]state.StorageItemWithKey, error) {
-	panic("TODO")
-}
-
 // CurrentHeaderHash implements Blockchainer interface.
 func (chain *FakeChain) CurrentHeaderHash() util.Uint256 {
 	return util.Uint256{}

pkg/core/blockchain.go

@@ -1589,11 +1589,6 @@ func (bc *Blockchain) GetStorageItem(id int32, key []byte) state.StorageItem {
 	return bc.dao.GetStorageItem(id, key)
 }
 
-// GetStorageItems returns all storage items for a given contract id.
-func (bc *Blockchain) GetStorageItems(id int32) ([]state.StorageItemWithKey, error) {
-	return bc.dao.GetStorageItems(id)
-}
-
 // GetBlock returns a Block by the given hash.
 func (bc *Blockchain) GetBlock(hash util.Uint256) (*block.Block, error) {
 	topBlock := bc.topBlock.Load()

pkg/core/blockchainer/blockchainer.go

@@ -59,7 +59,6 @@ type Blockchainer interface {
 	GetValidators() ([]*keys.PublicKey, error)
 	GetStateModule() StateRoot
 	GetStorageItem(id int32, key []byte) state.StorageItem
-	GetStorageItems(id int32) ([]state.StorageItemWithKey, error)
 	GetTestVM(t trigger.Type, tx *transaction.Transaction, b *block.Block) *interop.Context
 	GetTransaction(util.Uint256) (*transaction.Transaction, uint32, error)
 	SetOracle(service services.Oracle)

pkg/core/dao/dao.go

@@ -310,29 +310,6 @@ func (dao *Simple) DeleteStorageItem(id int32, key []byte) {
 	dao.Store.Delete(stKey)
 }
 
-// GetStorageItems returns all storage items for a given id.
-func (dao *Simple) GetStorageItems(id int32) ([]state.StorageItemWithKey, error) {
-	return dao.GetStorageItemsWithPrefix(id, nil)
-}
-
-// GetStorageItemsWithPrefix returns all storage items with given id for a
-// given scripthash.
-func (dao *Simple) GetStorageItemsWithPrefix(id int32, prefix []byte) ([]state.StorageItemWithKey, error) {
-	var siArr []state.StorageItemWithKey
-
-	saveToArr := func(k, v []byte) bool {
-		// Cut prefix and hash.
-		// #1468, but don't need to copy here, because it is done by Store.
-		siArr = append(siArr, state.StorageItemWithKey{
-			Key:  k,
-			Item: state.StorageItem(v),
-		})
-		return true
-	}
-	dao.Seek(id, storage.SeekRange{Prefix: prefix}, saveToArr)
-	return siArr, nil
-}
-
 // Seek executes f for all storage items matching a given `rng` (matching given prefix and
 // starting from the point specified). If key or value is to be used outside of f, they
 // may not be copied. Seek continues iterating until false is returned from f.

pkg/core/interop_system.go

@@ -189,8 +189,6 @@ func storageFind(ic *interop.Context) error {
 	if opts&istorage.FindDeserialize == 0 && (opts&istorage.FindPick0 != 0 || opts&istorage.FindPick1 != 0) {
 		return fmt.Errorf("%w: PickN is specified without Deserialize", errFindInvalidOptions)
 	}
-	// Items in seekres should be sorted by key, but GetStorageItemsWithPrefix returns
-	// sorted items, so no need to sort them one more time.
 	ctx, cancel := context.WithCancel(context.Background())
 	seekres := ic.DAO.SeekAsync(ctx, stc.ID, storage.SeekRange{Prefix: prefix})
 	item := istorage.NewIterator(seekres, prefix, opts)

pkg/core/native/designate.go

@@ -16,6 +16,7 @@ import (
 	"github.com/nspcc-dev/neo-go/pkg/core/native/noderoles"
 	"github.com/nspcc-dev/neo-go/pkg/core/state"
 	"github.com/nspcc-dev/neo-go/pkg/core/stateroot"
+	"github.com/nspcc-dev/neo-go/pkg/core/storage"
 	"github.com/nspcc-dev/neo-go/pkg/crypto/hash"
 	"github.com/nspcc-dev/neo-go/pkg/crypto/keys"
 	"github.com/nspcc-dev/neo-go/pkg/smartcontract"
@@ -253,35 +254,31 @@ func (s *Designate) GetDesignatedByRole(d *dao.Simple, r noderoles.Role, index u
 			return val.nodes.Copy(), val.height, nil
 		}
 	}
-	kvs, err := d.GetStorageItemsWithPrefix(s.ID, []byte{byte(r)})
-	if err != nil {
-		return nil, 0, err
-	}
 	var (
 		ns        NodeList
 		bestIndex uint32
-		resSi     state.StorageItem
+		resVal    []byte
+		start     = make([]byte, 4)
 	)
-	// kvs are sorted by key (BE index bytes) in ascending way, so iterate backwards to get the latest designated.
-	for i := len(kvs) - 1; i >= 0; i-- {
-		kv := kvs[i]
-		if len(kv.Key) < 4 {
-			continue
-		}
-		siInd := binary.BigEndian.Uint32(kv.Key)
-		if siInd <= index {
-			bestIndex = siInd
-			resSi = kv.Item
-			break
-		}
-	}
-	if resSi != nil {
-		err = stackitem.DeserializeConvertible(resSi, &ns)
+
+	binary.BigEndian.PutUint32(start, index)
+	d.Seek(s.ID, storage.SeekRange{
+		Prefix:    []byte{byte(r)},
+		Start:     start,
+		Backwards: true,
+	}, func(k, v []byte) bool {
+		bestIndex = binary.BigEndian.Uint32(k) // If len(k) < 4 the DB is broken and it deserves a panic.
+		resVal = v
+		// Take just the latest item, it's the one we need.
+		return false
+	})
+	if resVal != nil {
+		err := stackitem.DeserializeConvertible(resVal, &ns)
 		if err != nil {
 			return nil, 0, err
 		}
 	}
-	return keys.PublicKeys(ns), bestIndex, err
+	return keys.PublicKeys(ns), bestIndex, nil
 }
 
 func (s *Designate) designateAsRole(ic *interop.Context, args []stackitem.Item) stackitem.Item {

pkg/core/native/management.go

@@ -388,13 +388,11 @@ func (m *Management) Destroy(d *dao.Simple, hash util.Uint160) error {
 	key := MakeContractKey(hash)
 	d.DeleteStorageItem(m.ID, key)
 	d.DeleteContractID(contract.ID)
-	siArr, err := d.GetStorageItems(contract.ID)
-	if err != nil {
-		return err
-	}
-	for _, kv := range siArr {
-		d.DeleteStorageItem(contract.ID, []byte(kv.Key))
-	}
+
+	d.Seek(contract.ID, storage.SeekRange{}, func(k, _ []byte) bool {
+		d.DeleteStorageItem(contract.ID, k)
+		return true
+	})
 	m.markUpdated(hash)
 	return nil
 }

pkg/core/native/native_neo.go

@@ -255,11 +255,7 @@ func (n *NEO) InitializeCache(bc interop.Ledger, d *dao.Simple) error {
 		return err
 	}
 
-	gr, err := n.getSortedGASRecordFromDAO(d)
-	if err != nil {
-		return err
-	}
-	n.gasPerBlock.Store(gr)
+	n.gasPerBlock.Store(n.getSortedGASRecordFromDAO(d))
 	n.gasPerBlockChanged.Store(false)
 
 	return nil
@@ -375,11 +371,7 @@ func (n *NEO) PostPersist(ic *interop.Context) error {
 		}
 	}
 	if n.gasPerBlockChanged.Load().(bool) {
-		gr, err := n.getSortedGASRecordFromDAO(ic.DAO)
-		if err != nil {
-			panic(err)
-		}
-		n.gasPerBlock.Store(gr)
+		n.gasPerBlock.Store(n.getSortedGASRecordFromDAO(ic.DAO))
 		n.gasPerBlockChanged.Store(false)
 	}
 
@@ -495,35 +487,23 @@ func (n *NEO) getGASPerBlock(ic *interop.Context, _ []stackitem.Item) stackitem.
 	return stackitem.NewBigInteger(gas)
 }
 
-func (n *NEO) getSortedGASRecordFromDAO(d *dao.Simple) (gasRecord, error) {
-	grArr, err := d.GetStorageItemsWithPrefix(n.ID, []byte{prefixGASPerBlock})
-	if err != nil {
-		return gasRecord{}, fmt.Errorf("failed to get gas records from storage: %w", err)
-	}
-	var gr = make(gasRecord, len(grArr))
-	for i, kv := range grArr {
-		indexBytes, gasValue := kv.Key, kv.Item
-		gr[i] = gasIndexPair{
-			Index:       binary.BigEndian.Uint32([]byte(indexBytes)),
-			GASPerBlock: *bigint.FromBytes(gasValue),
-		}
-	}
-	// GAS records should be sorted by index, but GetStorageItemsWithPrefix returns
-	// values sorted by BE bytes of index, so we're OK with that.
-	return gr, nil
+func (n *NEO) getSortedGASRecordFromDAO(d *dao.Simple) gasRecord {
+	var gr = make(gasRecord, 0)
+	d.Seek(n.ID, storage.SeekRange{Prefix: []byte{prefixGASPerBlock}}, func(k, v []byte) bool {
+		gr = append(gr, gasIndexPair{
+			Index:       binary.BigEndian.Uint32(k),
+			GASPerBlock: *bigint.FromBytes(v),
+		})
+		return true
+	})
+	return gr
 }
 
 // GetGASPerBlock returns gas generated for block with provided index.
 func (n *NEO) GetGASPerBlock(d *dao.Simple, index uint32) *big.Int {
-	var (
-		gr  gasRecord
-		err error
-	)
+	var gr gasRecord
 	if n.gasPerBlockChanged.Load().(bool) {
-		gr, err = n.getSortedGASRecordFromDAO(d)
-		if err != nil {
-			panic(err)
-		}
+		gr = n.getSortedGASRecordFromDAO(d)
 	} else {
 		gr = n.gasPerBlock.Load().(gasRecord)
 	}
@@ -665,17 +645,11 @@ func (n *NEO) CalculateNEOHolderReward(d *dao.Simple, value *big.Int, start, end
 	} else if value.Sign() < 0 {
 		return nil, errors.New("negative value")
 	}
-	var (
-		gr  gasRecord
-		err error
-	)
+	var gr gasRecord
 	if !n.gasPerBlockChanged.Load().(bool) {
 		gr = n.gasPerBlock.Load().(gasRecord)
 	} else {
-		gr, err = n.getSortedGASRecordFromDAO(d)
-		if err != nil {
-			return nil, err
-		}
+		gr = n.getSortedGASRecordFromDAO(d)
 	}
 	var sum, tmp big.Int
 	for i := len(gr) - 1; i >= 0; i-- {
@@ -847,17 +821,15 @@ func (n *NEO) ModifyAccountVotes(acc *state.NEOBalance, d *dao.Simple, value *bi
 }
 
 func (n *NEO) getCandidates(d *dao.Simple, sortByKey bool) ([]keyWithVotes, error) {
-	siArr, err := d.GetStorageItemsWithPrefix(n.ID, []byte{prefixCandidate})
-	if err != nil {
-		return nil, err
-	}
-	arr := make([]keyWithVotes, 0, len(siArr))
-	for _, kv := range siArr {
-		c := new(candidate).FromBytes(kv.Item)
+	arr := make([]keyWithVotes, 0)
+	d.Seek(n.ID, storage.SeekRange{Prefix: []byte{prefixCandidate}}, func(k, v []byte) bool {
+		c := new(candidate).FromBytes(v)
 		if c.Registered {
-			arr = append(arr, keyWithVotes{Key: string(kv.Key), Votes: &c.Votes})
+			arr = append(arr, keyWithVotes{Key: string(k), Votes: &c.Votes})
 		}
-	}
+		return true
+	})
+
 	if !sortByKey {
 		// sortByKey assumes to sort by serialized key bytes (that's the way keys
 		// are stored and retrieved from the storage by default). Otherwise, need

pkg/core/native/oracle.go

@@ -471,22 +471,24 @@ func (o *Oracle) getOriginalTxID(d *dao.Simple, tx *transaction.Transaction) uti
 
 // getRequests returns all requests which have not been finished yet.
 func (o *Oracle) getRequests(d *dao.Simple) (map[uint64]*state.OracleRequest, error) {
-	arr, err := d.GetStorageItemsWithPrefix(o.ID, prefixRequest)
-	if err != nil {
-		return nil, err
-	}
-	reqs := make(map[uint64]*state.OracleRequest, len(arr))
-	for _, kv := range arr {
-		if len(kv.Key) != 8 {
-			return nil, errors.New("invalid request ID")
+	var reqs = make(map[uint64]*state.OracleRequest)
+	var err error
+	d.Seek(o.ID, storage.SeekRange{Prefix: prefixRequest}, func(k, v []byte) bool {
+		if len(k) != 8 {
+			err = errors.New("invalid request ID")
+			return false
 		}
 		req := new(state.OracleRequest)
-		err = stackitem.DeserializeConvertible(kv.Item, req)
+		err = stackitem.DeserializeConvertible(v, req)
 		if err != nil {
-			return nil, err
+			return false
 		}
-		id := binary.BigEndian.Uint64([]byte(kv.Key))
+		id := binary.BigEndian.Uint64(k)
 		reqs[id] = req
+		return true
+	})
+	if err != nil {
+		return nil, err
 	}
 	return reqs, nil
 }

pkg/core/native/policy.go

@@ -10,6 +10,7 @@ import (
 	"github.com/nspcc-dev/neo-go/pkg/core/interop"
 	"github.com/nspcc-dev/neo-go/pkg/core/native/nativenames"
 	"github.com/nspcc-dev/neo-go/pkg/core/state"
+	"github.com/nspcc-dev/neo-go/pkg/core/storage"
 	"github.com/nspcc-dev/neo-go/pkg/core/transaction"
 	"github.com/nspcc-dev/neo-go/pkg/smartcontract"
 	"github.com/nspcc-dev/neo-go/pkg/smartcontract/callflag"
@@ -156,23 +157,20 @@ func (p *Policy) PostPersist(ic *interop.Context) error {
 	p.storagePrice = uint32(getIntWithKey(p.ID, ic.DAO, storagePriceKey))
 
 	p.blockedAccounts = make([]util.Uint160, 0)
-	siArr, err := ic.DAO.GetStorageItemsWithPrefix(p.ID, []byte{blockedAccountPrefix})
-	if err != nil {
-		return fmt.Errorf("failed to get blocked accounts from storage: %w", err)
-	}
-	for _, kv := range siArr {
-		hash, err := util.Uint160DecodeBytesBE([]byte(kv.Key))
+	var fErr error
+	ic.DAO.Seek(p.ID, storage.SeekRange{Prefix: []byte{blockedAccountPrefix}}, func(k, _ []byte) bool {
+		hash, err := util.Uint160DecodeBytesBE(k)
 		if err != nil {
-			return fmt.Errorf("failed to decode blocked account hash: %w", err)
+			fErr = fmt.Errorf("failed to decode blocked account hash: %w", err)
+			return false
 		}
 		p.blockedAccounts = append(p.blockedAccounts, hash)
+		return true
+	})
+	if fErr == nil {
+		p.isValid = true
 	}
-	// blockedAccounts should be sorted by account BE bytes, but GetStorageItemsWithPrefix
-	// returns values sorted by key (which is account's BE bytes), so don't need to sort
-	// one more time.
-
-	p.isValid = true
-	return nil
+	return fErr
 }
 
 // getFeePerByte is Policy contract method and returns required transaction's fee

pkg/core/state/storage_item.go

@@ -2,9 +2,3 @@ package state
 
 // StorageItem is the value to be stored with read-only flag.
 type StorageItem []byte
-
-// StorageItemWithKey is a storage item with corresponding key.
-type StorageItemWithKey struct {
-	Key  []byte
-	Item StorageItem
-}