dstepanov-yadro/neoneo-go
1
0
Fork 0
forked from TrueCloudLab/neoneo-go
Code Pull requests Activity

Merge pull request #2364 from nspcc-dev/botch

Browse source 
storage: speedups and simplifications
This commit is contained in:
Roman Khimov 2022-02-18 14:11:40 +03:00 committed by GitHub
commit baa05bef43
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
38 changed files with 718 additions and 1078 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
pkg/compiler/interop_test.go
View file

@ -178,7 +178,7 @@ func TestAppCall(t *testing.T) {
require.NoError(t, err) require.NoError(t, err)
ih := hash.Hash160(inner.Script) ih := hash.Hash160(inner.Script)
var contractGetter = func(_ dao.DAO, h util.Uint160) (*state.Contract, error) { var contractGetter = func(_ *dao.Simple, h util.Uint160) (*state.Contract, error) {
if h.Equals(ih) { if h.Equals(ih) {
return &state.Contract{ return &state.Contract{
ContractBase: state.ContractBase{ ContractBase: state.ContractBase{

132
pkg/core/blockchain.go
View file

@ -20,6 +20,7 @@ import (
"github.com/nspcc-dev/neo-go/pkg/core/interop" "github.com/nspcc-dev/neo-go/pkg/core/interop"
"github.com/nspcc-dev/neo-go/pkg/core/interop/contract" "github.com/nspcc-dev/neo-go/pkg/core/interop/contract"
"github.com/nspcc-dev/neo-go/pkg/core/mempool" "github.com/nspcc-dev/neo-go/pkg/core/mempool"
"github.com/nspcc-dev/neo-go/pkg/core/mpt"
"github.com/nspcc-dev/neo-go/pkg/core/native" "github.com/nspcc-dev/neo-go/pkg/core/native"
"github.com/nspcc-dev/neo-go/pkg/core/native/noderoles" "github.com/nspcc-dev/neo-go/pkg/core/native/noderoles"
"github.com/nspcc-dev/neo-go/pkg/core/state" "github.com/nspcc-dev/neo-go/pkg/core/state"
@ -316,9 +317,7 @@ func (bc *Blockchain) init() error {
KeepOnlyLatestState: bc.config.KeepOnlyLatestState, KeepOnlyLatestState: bc.config.KeepOnlyLatestState,
Value: version, Value: version,
} }
if err = bc.dao.PutVersion(ver); err != nil { bc.dao.PutVersion(ver)
return err
}
bc.dao.Version = ver bc.dao.Version = ver
bc.persistent.Version = ver bc.persistent.Version = ver
genesisBlock, err := createGenesisBlock(bc.config) genesisBlock, err := createGenesisBlock(bc.config)
@ -326,10 +325,7 @@ func (bc *Blockchain) init() error {
return err return err
} }
bc.headerHashes = []util.Uint256{genesisBlock.Hash()} bc.headerHashes = []util.Uint256{genesisBlock.Hash()}
err = bc.dao.PutCurrentHeader(hashAndIndexToBytes(genesisBlock.Hash(), genesisBlock.Index)) bc.dao.PutCurrentHeader(hashAndIndexToBytes(genesisBlock.Hash(), genesisBlock.Index))
if err != nil {
return err
}
if err := bc.stateRoot.Init(0); err != nil { if err := bc.stateRoot.Init(0); err != nil {
return fmt.Errorf("can't init MPT: %w", err) return fmt.Errorf("can't init MPT: %w", err)
} }
@ -504,14 +500,10 @@ func (bc *Blockchain) jumpToStateInternal(p uint32, stage stateJumpStage) error
bc.log.Info("jumping to state sync point", zap.Uint32("state sync point", p)) bc.log.Info("jumping to state sync point", zap.Uint32("state sync point", p))
writeBuf := io.NewBufBinWriter()
jumpStageKey := storage.SYSStateJumpStage.Bytes() jumpStageKey := storage.SYSStateJumpStage.Bytes()
switch stage { switch stage {
case none: case none:
err := bc.dao.Store.Put(jumpStageKey, []byte{byte(stateJumpStarted)}) bc.dao.Store.Put(jumpStageKey, []byte{byte(stateJumpStarted)})
if err != nil {
return fmt.Errorf("failed to store state jump stage: %w", err)
}
fallthrough fallthrough
case stateJumpStarted: case stateJumpStarted:
newPrefix := statesync.TemporaryPrefix(bc.dao.Version.StoragePrefix) newPrefix := statesync.TemporaryPrefix(bc.dao.Version.StoragePrefix)
@ -520,55 +512,40 @@ func (bc *Blockchain) jumpToStateInternal(p uint32, stage stateJumpStage) error
return fmt.Errorf("failed to get dao.Version: %w", err) return fmt.Errorf("failed to get dao.Version: %w", err)
} }
v.StoragePrefix = newPrefix v.StoragePrefix = newPrefix
if err := bc.dao.PutVersion(v); err != nil { bc.dao.PutVersion(v)
return fmt.Errorf("failed to update dao.Version: %w", err)
}
bc.persistent.Version = v bc.persistent.Version = v
err = bc.dao.Store.Put(jumpStageKey, []byte{byte(newStorageItemsAdded)}) bc.dao.Store.Put(jumpStageKey, []byte{byte(newStorageItemsAdded)})
if err != nil {
return fmt.Errorf("failed to store state jump stage: %w", err)
}
fallthrough fallthrough
case newStorageItemsAdded: case newStorageItemsAdded:
b := bc.dao.Store.Batch() cache := bc.dao.GetPrivate()
prefix := statesync.TemporaryPrefix(bc.dao.Version.StoragePrefix) prefix := statesync.TemporaryPrefix(bc.dao.Version.StoragePrefix)
bc.dao.Store.Seek(storage.SeekRange{Prefix: []byte{byte(prefix)}}, func(k, _ []byte) bool { bc.dao.Store.Seek(storage.SeekRange{Prefix: []byte{byte(prefix)}}, func(k, _ []byte) bool {
// #1468, but don't need to copy here, because it is done by Store. // #1468, but don't need to copy here, because it is done by Store.
b.Delete(k) cache.Store.Delete(k)
return true return true
}) })
err := bc.dao.Store.PutBatch(b)
if err != nil {
return fmt.Errorf("failed to remove old storage items: %w", err)
}
// After current state is updated, we need to remove outdated state-related data if so. // After current state is updated, we need to remove outdated state-related data if so.
// The only outdated data we might have is genesis-related data, so check it. // The only outdated data we might have is genesis-related data, so check it.
if p-bc.config.MaxTraceableBlocks > 0 { if p-bc.config.MaxTraceableBlocks > 0 {
cache := bc.dao.GetWrapped().(*dao.Simple) err := cache.DeleteBlock(bc.headerHashes[0])
writeBuf.Reset()
err := cache.DeleteBlock(bc.headerHashes[0], writeBuf)
if err != nil { if err != nil {
return fmt.Errorf("failed to remove outdated state data for the genesis block: %w", err) return fmt.Errorf("failed to remove outdated state data for the genesis block: %w", err)
} }
prefixes := []byte{byte(storage.STNEP11Transfers), byte(storage.STNEP17Transfers), byte(storage.STTokenTransferInfo)} prefixes := []byte{byte(storage.STNEP11Transfers), byte(storage.STNEP17Transfers), byte(storage.STTokenTransferInfo)}
for i := range prefixes { for i := range prefixes {
cache.Store.Seek(storage.SeekRange{Prefix: prefixes[i : i+1]}, func(k, v []byte) bool { cache.Store.Seek(storage.SeekRange{Prefix: prefixes[i : i+1]}, func(k, v []byte) bool {
_ = cache.Store.Delete(k) // It's MemCachedStore which never returns an error. cache.Store.Delete(k)
return true return true
}) })
} }
_, err = cache.Persist()
if err != nil {
return fmt.Errorf("failed to drop genesis block state: %w", err)
} }
} cache.Store.Put(jumpStageKey, []byte{byte(genesisStateRemoved)})
err = bc.dao.Store.Put(jumpStageKey, []byte{byte(genesisStateRemoved)}) _, err := cache.Persist()
if err != nil { if err != nil {
return fmt.Errorf("failed to store state jump stage: %w", err) return fmt.Errorf("failed to persist old items removal: %w", err)
} }
case genesisStateRemoved: case genesisStateRemoved:
// there's nothing to do after that, so just continue with common operations // there's nothing to do after that, so just continue with common operations
@ -581,11 +558,7 @@ func (bc *Blockchain) jumpToStateInternal(p uint32, stage stateJumpStage) error
if err != nil { if err != nil {
return fmt.Errorf("failed to get current block: %w", err) return fmt.Errorf("failed to get current block: %w", err)
} }
writeBuf.Reset() bc.dao.StoreAsCurrentBlock(block)
err = bc.dao.StoreAsCurrentBlock(block, writeBuf)
if err != nil {
return fmt.Errorf("failed to store current block: %w", err)
}
bc.topBlock.Store(block) bc.topBlock.Store(block)
atomic.StoreUint32(&bc.blockHeight, p) atomic.StoreUint32(&bc.blockHeight, p)
atomic.StoreUint32(&bc.persistedHeight, p) atomic.StoreUint32(&bc.persistedHeight, p)
@ -594,12 +567,10 @@ func (bc *Blockchain) jumpToStateInternal(p uint32, stage stateJumpStage) error
if err != nil { if err != nil {
return fmt.Errorf("failed to get block to init MPT: %w", err) return fmt.Errorf("failed to get block to init MPT: %w", err)
} }
if err = bc.stateRoot.JumpToState(&state.MPTRoot{ bc.stateRoot.JumpToState(&state.MPTRoot{
Index: p, Index: p,
Root: block.PrevStateRoot, Root: block.PrevStateRoot,
}); err != nil { })
return fmt.Errorf("can't perform MPT jump to height %d: %w", p, err)
}
err = bc.contracts.NEO.InitializeCache(bc, bc.dao) err = bc.contracts.NEO.InitializeCache(bc, bc.dao)
if err != nil { if err != nil {
@ -617,10 +588,7 @@ func (bc *Blockchain) jumpToStateInternal(p uint32, stage stateJumpStage) error
updateBlockHeightMetric(p) updateBlockHeightMetric(p)
err = bc.dao.Store.Delete(jumpStageKey) bc.dao.Store.Delete(jumpStageKey)
if err != nil {
return fmt.Errorf("failed to remove outdated state jump stage: %w", err)
}
return nil return nil
} }
@ -933,7 +901,7 @@ func (bc *Blockchain) AddHeaders(headers ...*block.Header) error {
func (bc *Blockchain) addHeaders(verify bool, headers ...*block.Header) error { func (bc *Blockchain) addHeaders(verify bool, headers ...*block.Header) error {
var ( var (
start = time.Now() start = time.Now()
batch = bc.dao.Store.Batch() batch = bc.dao.GetPrivate()
err error err error
) )
@ -982,7 +950,7 @@ func (bc *Blockchain) addHeaders(verify bool, headers ...*block.Header) error {
} }
key := storage.AppendPrefix(storage.DataExecutable, h.Hash().BytesBE()) key := storage.AppendPrefix(storage.DataExecutable, h.Hash().BytesBE())
batch.Put(key, buf.Bytes()) batch.Store.Put(key, buf.Bytes())
buf.Reset() buf.Reset()
lastHeader = h lastHeader = h
} }
@ -995,13 +963,13 @@ func (bc *Blockchain) addHeaders(verify bool, headers ...*block.Header) error {
} }
key := storage.AppendPrefixInt(storage.IXHeaderHashList, int(bc.storedHeaderCount)) key := storage.AppendPrefixInt(storage.IXHeaderHashList, int(bc.storedHeaderCount))
batch.Put(key, buf.Bytes()) batch.Store.Put(key, buf.Bytes())
bc.storedHeaderCount += headerBatchCount bc.storedHeaderCount += headerBatchCount
} }
batch.Put(storage.SYSCurrentHeader.Bytes(), hashAndIndexToBytes(lastHeader.Hash(), lastHeader.Index)) batch.Store.Put(storage.SYSCurrentHeader.Bytes(), hashAndIndexToBytes(lastHeader.Hash(), lastHeader.Index))
updateHeaderHeightMetric(len(bc.headerHashes) - 1) updateHeaderHeightMetric(len(bc.headerHashes) - 1)
if err = bc.dao.Store.PutBatch(batch); err != nil { if _, err = batch.Persist(); err != nil {
return err return err
} }
bc.log.Debug("done processing headers", bc.log.Debug("done processing headers",
@ -1027,8 +995,8 @@ func (bc *Blockchain) GetStateSyncModule() *statesync.Module {
// This is the only way to change Blockchain state. // This is the only way to change Blockchain state.
func (bc *Blockchain) storeBlock(block *block.Block, txpool *mempool.Pool) error { func (bc *Blockchain) storeBlock(block *block.Block, txpool *mempool.Pool) error {
var ( var (
cache = bc.dao.GetWrapped() cache = bc.dao.GetPrivate()
aerCache = bc.dao.GetWrapped() aerCache = bc.dao.GetPrivate()
appExecResults = make([]*state.AppExecResult, 0, 2+len(block.Transactions)) appExecResults = make([]*state.AppExecResult, 0, 2+len(block.Transactions))
aerchan = make(chan *state.AppExecResult, len(block.Transactions)/8) // Tested 8 and 4 with no practical difference, but feel free to test more and tune. aerchan = make(chan *state.AppExecResult, len(block.Transactions)/8) // Tested 8 and 4 with no practical difference, but feel free to test more and tune.
aerdone = make(chan error) aerdone = make(chan error)
@ -1036,17 +1004,12 @@ func (bc *Blockchain) storeBlock(block *block.Block, txpool *mempool.Pool) error
go func() { go func() {
var ( var (
kvcache = aerCache kvcache = aerCache
writeBuf = io.NewBufBinWriter()
err error err error
txCnt int txCnt int
baer1, baer2 *state.AppExecResult baer1, baer2 *state.AppExecResult
transCache = make(map[util.Uint160]transferData) transCache = make(map[util.Uint160]transferData)
) )
if err := kvcache.StoreAsCurrentBlock(block, writeBuf); err != nil { kvcache.StoreAsCurrentBlock(block)
aerdone <- err
return
}
writeBuf.Reset()
if bc.config.RemoveUntraceableBlocks { if bc.config.RemoveUntraceableBlocks {
var start, stop uint32 var start, stop uint32
if bc.config.P2PStateExchangeExtensions { if bc.config.P2PStateExchangeExtensions {
@ -1064,13 +1027,12 @@ func (bc *Blockchain) storeBlock(block *block.Block, txpool *mempool.Pool) error
stop = start + 1 stop = start + 1
} }
for index := start; index < stop; index++ { for index := start; index < stop; index++ {
err := kvcache.DeleteBlock(bc.headerHashes[index], writeBuf) err := kvcache.DeleteBlock(bc.headerHashes[index])
if err != nil { if err != nil {
bc.log.Warn("error while removing old block", bc.log.Warn("error while removing old block",
zap.Uint32("index", index), zap.Uint32("index", index),
zap.Error(err)) zap.Error(err))
} }
writeBuf.Reset()
} }
} }
for aer := range aerchan { for aer := range aerchan {
@ -1081,7 +1043,7 @@ func (bc *Blockchain) storeBlock(block *block.Block, txpool *mempool.Pool) error
baer2 = aer baer2 = aer
} }
} else { } else {
err = kvcache.StoreAsTransaction(block.Transactions[txCnt], block.Index, aer, writeBuf) err = kvcache.StoreAsTransaction(block.Transactions[txCnt], block.Index, aer)
txCnt++ txCnt++
} }
if err != nil { if err != nil {
@ -1093,17 +1055,15 @@ func (bc *Blockchain) storeBlock(block *block.Block, txpool *mempool.Pool) error
bc.handleNotification(&aer.Execution.Events[j], kvcache, transCache, block, aer.Container) bc.handleNotification(&aer.Execution.Events[j], kvcache, transCache, block, aer.Container)
} }
} }
writeBuf.Reset()
} }
if err != nil { if err != nil {
aerdone <- err aerdone <- err
return return
} }
if err := kvcache.StoreAsBlock(block, baer1, baer2, writeBuf); err != nil { if err := kvcache.StoreAsBlock(block, baer1, baer2); err != nil {
aerdone <- err aerdone <- err
return return
} }
writeBuf.Reset()
for acc, trData := range transCache { for acc, trData := range transCache {
err = kvcache.PutTokenTransferInfo(acc, &trData.Info) err = kvcache.PutTokenTransferInfo(acc, &trData.Info)
if err != nil { if err != nil {
@ -1111,18 +1071,10 @@ func (bc *Blockchain) storeBlock(block *block.Block, txpool *mempool.Pool) error
return return
} }
if !trData.Info.NewNEP11Batch { if !trData.Info.NewNEP11Batch {
err = kvcache.PutTokenTransferLog(acc, trData.Info.NextNEP11NewestTimestamp, trData.Info.NextNEP11Batch, true, &trData.Log11) kvcache.PutTokenTransferLog(acc, trData.Info.NextNEP11NewestTimestamp, trData.Info.NextNEP11Batch, true, &trData.Log11)
if err != nil {
aerdone <- err
return
}
} }
if !trData.Info.NewNEP17Batch { if !trData.Info.NewNEP17Batch {
err = kvcache.PutTokenTransferLog(acc, trData.Info.NextNEP17NewestTimestamp, trData.Info.NextNEP17Batch, false, &trData.Log17) kvcache.PutTokenTransferLog(acc, trData.Info.NextNEP17NewestTimestamp, trData.Info.NextNEP17Batch, false, &trData.Log17)
if err != nil {
aerdone <- err
return
}
} }
} }
close(aerdone) close(aerdone)
@ -1187,9 +1139,8 @@ func (bc *Blockchain) storeBlock(block *block.Block, txpool *mempool.Pool) error
appExecResults = append(appExecResults, aer) appExecResults = append(appExecResults, aer)
aerchan <- aer aerchan <- aer
close(aerchan) close(aerchan)
d := cache.(*dao.Simple) b := mpt.MapToMPTBatch(cache.Store.GetStorageChanges())
b := d.GetMPTBatch() mpt, sr, err := bc.stateRoot.AddMPTBatch(block.Index, b, cache.Store)
mpt, sr, err := bc.stateRoot.AddMPTBatch(block.Index, b, d.Store)
if err != nil { if err != nil {
// Release goroutines, don't care about errors, we already have one. // Release goroutines, don't care about errors, we already have one.
<-aerdone <-aerdone
@ -1213,7 +1164,7 @@ func (bc *Blockchain) storeBlock(block *block.Block, txpool *mempool.Pool) error
} }
if bc.config.SaveStorageBatch { if bc.config.SaveStorageBatch {
bc.lastBatch = d.GetBatch() bc.lastBatch = cache.GetBatch()
} }
// Every persist cycle we also compact our in-memory MPT. It's flushed // Every persist cycle we also compact our in-memory MPT. It's flushed
// already in AddMPTBatch, so collapsing it is safe. // already in AddMPTBatch, so collapsing it is safe.
@ -1313,7 +1264,7 @@ func (bc *Blockchain) IsExtensibleAllowed(u util.Uint160) bool {
return n < len(us) return n < len(us)
} }
func (bc *Blockchain) runPersist(script []byte, block *block.Block, cache dao.DAO, trig trigger.Type) (*state.AppExecResult, error) { func (bc *Blockchain) runPersist(script []byte, block *block.Block, cache *dao.Simple, trig trigger.Type) (*state.AppExecResult, error) {
systemInterop := bc.newInteropContext(trig, cache, block, nil) systemInterop := bc.newInteropContext(trig, cache, block, nil)
v := systemInterop.SpawnVM() v := systemInterop.SpawnVM()
v.LoadScriptWithFlags(script, callflag.All) v.LoadScriptWithFlags(script, callflag.All)
@ -1335,7 +1286,7 @@ func (bc *Blockchain) runPersist(script []byte, block *block.Block, cache dao.DA
}, nil }, nil
} }
func (bc *Blockchain) handleNotification(note *state.NotificationEvent, d dao.DAO, func (bc *Blockchain) handleNotification(note *state.NotificationEvent, d *dao.Simple,
transCache map[util.Uint160]transferData, b *block.Block, h util.Uint256) { transCache map[util.Uint160]transferData, b *block.Block, h util.Uint256) {
if note.Name != "Transfer" { if note.Name != "Transfer" {
return return
@ -1378,7 +1329,7 @@ func parseUint160(itm stackitem.Item) (util.Uint160, error) {
return util.Uint160DecodeBytesBE(bytes) return util.Uint160DecodeBytesBE(bytes)
} }
func (bc *Blockchain) processTokenTransfer(cache dao.DAO, transCache map[util.Uint160]transferData, func (bc *Blockchain) processTokenTransfer(cache *dao.Simple, transCache map[util.Uint160]transferData,
h util.Uint256, b *block.Block, sc util.Uint160, from util.Uint160, to util.Uint160, h util.Uint256, b *block.Block, sc util.Uint160, from util.Uint160, to util.Uint160,
amount *big.Int, tokenID []byte) { amount *big.Int, tokenID []byte) {
var id int32 var id int32
@ -1432,7 +1383,7 @@ func (bc *Blockchain) processTokenTransfer(cache dao.DAO, transCache map[util.Ui
} }
} }
func appendTokenTransfer(cache dao.DAO, transCache map[util.Uint160]transferData, addr util.Uint160, transfer io.Serializable, func appendTokenTransfer(cache *dao.Simple, transCache map[util.Uint160]transferData, addr util.Uint160, transfer io.Serializable,
token int32, bIndex uint32, bTimestamp uint64, isNEP11 bool) error { token int32, bIndex uint32, bTimestamp uint64, isNEP11 bool) error {
transferData, ok := transCache[addr] transferData, ok := transCache[addr]
if !ok { if !ok {
@ -1480,10 +1431,7 @@ func appendTokenTransfer(cache dao.DAO, transCache map[util.Uint160]transferData
transferData.Info.LastUpdated[token] = bIndex transferData.Info.LastUpdated[token] = bIndex
*newBatch = log.Size() >= state.TokenTransferBatchSize *newBatch = log.Size() >= state.TokenTransferBatchSize
if *newBatch { if *newBatch {
err = cache.PutTokenTransferLog(addr, *currTimestamp, *nextBatch, isNEP11, log) cache.PutTokenTransferLog(addr, *currTimestamp, *nextBatch, isNEP11, log)
if err != nil {
return err
}
*nextBatch++ *nextBatch++
*currTimestamp = bTimestamp *currTimestamp = bTimestamp
// Put makes a copy of it anyway. // Put makes a copy of it anyway.
@ -2198,7 +2146,7 @@ func (bc *Blockchain) GetEnrollments() ([]state.Validator, error) {
// GetTestVM returns an interop context with VM set up for a test run. // GetTestVM returns an interop context with VM set up for a test run.
func (bc *Blockchain) GetTestVM(t trigger.Type, tx *transaction.Transaction, b *block.Block) *interop.Context { func (bc *Blockchain) GetTestVM(t trigger.Type, tx *transaction.Transaction, b *block.Block) *interop.Context {
d := bc.dao.GetWrapped().(*dao.Simple) d := bc.dao.GetPrivate()
systemInterop := bc.newInteropContext(t, d, b, tx) systemInterop := bc.newInteropContext(t, d, b, tx)
vm := systemInterop.SpawnVM() vm := systemInterop.SpawnVM()
vm.SetPriceGetter(systemInterop.GetPrice) vm.SetPriceGetter(systemInterop.GetPrice)
@ -2368,7 +2316,7 @@ func hashAndIndexToBytes(h util.Uint256, index uint32) []byte {
return buf.Bytes() return buf.Bytes()
} }
func (bc *Blockchain) newInteropContext(trigger trigger.Type, d dao.DAO, block *block.Block, tx *transaction.Transaction) *interop.Context { func (bc *Blockchain) newInteropContext(trigger trigger.Type, d *dao.Simple, block *block.Block, tx *transaction.Transaction) *interop.Context {
ic := interop.NewContext(trigger, bc, d, bc.contracts.Management.GetContract, bc.contracts.Contracts, block, tx, bc.log) ic := interop.NewContext(trigger, bc, d, bc.contracts.Management.GetContract, bc.contracts.Contracts, block, tx, bc.log)
ic.Functions = systemInterops ic.Functions = systemInterops
switch { switch {

31
pkg/core/blockchain_test.go
View file

@ -804,7 +804,7 @@ func TestVerifyTx(t *testing.T) {
}, },
}, },
} }
require.NoError(t, bc.dao.StoreAsTransaction(conflicting, bc.blockHeight, nil, nil)) require.NoError(t, bc.dao.StoreAsTransaction(conflicting, bc.blockHeight, nil))
require.True(t, errors.Is(bc.VerifyTx(tx), ErrHasConflicts)) require.True(t, errors.Is(bc.VerifyTx(tx), ErrHasConflicts))
}) })
t.Run("attribute on-chain conflict", func(t *testing.T) { t.Run("attribute on-chain conflict", func(t *testing.T) {
@ -1385,7 +1385,7 @@ func TestClose(t *testing.T) {
// It's a hack, but we use internal knowledge of MemoryStore // It's a hack, but we use internal knowledge of MemoryStore
// implementation which makes it completely unusable (up to panicing) // implementation which makes it completely unusable (up to panicing)
// after Close(). // after Close().
_ = bc.dao.Store.Put([]byte{0}, []byte{1}) bc.dao.Store.Put([]byte{0}, []byte{1})
// This should never be executed. // This should never be executed.
assert.Nil(t, t) assert.Nil(t, t)
@ -1602,16 +1602,16 @@ func TestRemoveOldTransfers(t *testing.T) {
ttl := state.TokenTransferLog{Raw: []byte{1}} // It's incorrect, but who cares. ttl := state.TokenTransferLog{Raw: []byte{1}} // It's incorrect, but who cares.
for i := uint32(0); i < 3; i++ { for i := uint32(0); i < 3; i++ {
require.NoError(t, bc.dao.PutTokenTransferLog(acc1, older, i, false, &ttl)) bc.dao.PutTokenTransferLog(acc1, older, i, false, &ttl)
} }
for i := uint32(0); i < 3; i++ { for i := uint32(0); i < 3; i++ {
require.NoError(t, bc.dao.PutTokenTransferLog(acc2, newer, i, false, &ttl)) bc.dao.PutTokenTransferLog(acc2, newer, i, false, &ttl)
} }
for i := uint32(0); i < 2; i++ { for i := uint32(0); i < 2; i++ {
require.NoError(t, bc.dao.PutTokenTransferLog(acc3, older, i, true, &ttl)) bc.dao.PutTokenTransferLog(acc3, older, i, true, &ttl)
} }
for i := uint32(0); i < 2; i++ { for i := uint32(0); i < 2; i++ {
require.NoError(t, bc.dao.PutTokenTransferLog(acc3, newer, i, true, &ttl)) bc.dao.PutTokenTransferLog(acc3, newer, i, true, &ttl)
} }
_, err = bc.dao.Persist() _, err = bc.dao.Persist()
@ -1846,7 +1846,7 @@ func TestBlockchain_InitWithIncompleteStateJump(t *testing.T) {
require.NoError(t, bcSpout.AddHeaders(&b.Header)) require.NoError(t, bcSpout.AddHeaders(&b.Header))
// put storage items with STTemp prefix // put storage items with STTemp prefix
batch := bcSpout.dao.Store.Batch() batch := storage.NewMemCachedStore(bcSpout.dao.Store)
tempPrefix := storage.STTempStorage tempPrefix := storage.STTempStorage
if bcSpout.dao.Version.StoragePrefix == tempPrefix { if bcSpout.dao.Version.StoragePrefix == tempPrefix {
tempPrefix = storage.STStorage tempPrefix = storage.STStorage
@ -1858,7 +1858,8 @@ func TestBlockchain_InitWithIncompleteStateJump(t *testing.T) {
batch.Put(key, value) batch.Put(key, value)
return true return true
}) })
require.NoError(t, bcSpout.dao.Store.PutBatch(batch)) _, err := batch.Persist()
require.NoError(t, err)
checkNewBlockchainErr := func(t *testing.T, cfg func(c *config.Config), store storage.Store, shouldFail bool) { checkNewBlockchainErr := func(t *testing.T, cfg func(c *config.Config), store storage.Store, shouldFail bool) {
unitTestNetCfg, err := config.Load("../../config", testchain.Network()) unitTestNetCfg, err := config.Load("../../config", testchain.Network())
@ -1878,33 +1879,33 @@ func TestBlockchain_InitWithIncompleteStateJump(t *testing.T) {
} }
// manually store statejump stage to check statejump recover process // manually store statejump stage to check statejump recover process
t.Run("invalid RemoveUntraceableBlocks setting", func(t *testing.T) { t.Run("invalid RemoveUntraceableBlocks setting", func(t *testing.T) {
require.NoError(t, bcSpout.dao.Store.Put(storage.SYSStateJumpStage.Bytes(), []byte{byte(stateJumpStarted)})) bcSpout.dao.Store.Put(storage.SYSStateJumpStage.Bytes(), []byte{byte(stateJumpStarted)})
checkNewBlockchainErr(t, func(c *config.Config) { checkNewBlockchainErr(t, func(c *config.Config) {
boltCfg(c) boltCfg(c)
c.ProtocolConfiguration.RemoveUntraceableBlocks = false c.ProtocolConfiguration.RemoveUntraceableBlocks = false
}, bcSpout.dao.Store, true) }, bcSpout.dao.Store, true)
}) })
t.Run("invalid state jump stage format", func(t *testing.T) { t.Run("invalid state jump stage format", func(t *testing.T) {
require.NoError(t, bcSpout.dao.Store.Put(storage.SYSStateJumpStage.Bytes(), []byte{0x01, 0x02})) bcSpout.dao.Store.Put(storage.SYSStateJumpStage.Bytes(), []byte{0x01, 0x02})
checkNewBlockchainErr(t, boltCfg, bcSpout.dao.Store, true) checkNewBlockchainErr(t, boltCfg, bcSpout.dao.Store, true)
}) })
t.Run("missing state sync point", func(t *testing.T) { t.Run("missing state sync point", func(t *testing.T) {
require.NoError(t, bcSpout.dao.Store.Put(storage.SYSStateJumpStage.Bytes(), []byte{byte(stateJumpStarted)})) bcSpout.dao.Store.Put(storage.SYSStateJumpStage.Bytes(), []byte{byte(stateJumpStarted)})
checkNewBlockchainErr(t, boltCfg, bcSpout.dao.Store, true) checkNewBlockchainErr(t, boltCfg, bcSpout.dao.Store, true)
}) })
t.Run("invalid state sync point", func(t *testing.T) { t.Run("invalid state sync point", func(t *testing.T) {
require.NoError(t, bcSpout.dao.Store.Put(storage.SYSStateJumpStage.Bytes(), []byte{byte(stateJumpStarted)})) bcSpout.dao.Store.Put(storage.SYSStateJumpStage.Bytes(), []byte{byte(stateJumpStarted)})
point := make([]byte, 4) point := make([]byte, 4)
binary.LittleEndian.PutUint32(point, uint32(len(bcSpout.headerHashes))) binary.LittleEndian.PutUint32(point, uint32(len(bcSpout.headerHashes)))
require.NoError(t, bcSpout.dao.Store.Put(storage.SYSStateSyncPoint.Bytes(), point)) bcSpout.dao.Store.Put(storage.SYSStateSyncPoint.Bytes(), point)
checkNewBlockchainErr(t, boltCfg, bcSpout.dao.Store, true) checkNewBlockchainErr(t, boltCfg, bcSpout.dao.Store, true)
}) })
for _, stage := range []stateJumpStage{stateJumpStarted, newStorageItemsAdded, genesisStateRemoved, 0x03} { for _, stage := range []stateJumpStage{stateJumpStarted, newStorageItemsAdded, genesisStateRemoved, 0x03} {
t.Run(fmt.Sprintf("state jump stage %d", stage), func(t *testing.T) { t.Run(fmt.Sprintf("state jump stage %d", stage), func(t *testing.T) {
require.NoError(t, bcSpout.dao.Store.Put(storage.SYSStateJumpStage.Bytes(), []byte{byte(stage)})) bcSpout.dao.Store.Put(storage.SYSStateJumpStage.Bytes(), []byte{byte(stage)})
point := make([]byte, 4) point := make([]byte, 4)
binary.LittleEndian.PutUint32(point, uint32(stateSyncPoint)) binary.LittleEndian.PutUint32(point, uint32(stateSyncPoint))
require.NoError(t, bcSpout.dao.Store.Put(storage.SYSStateSyncPoint.Bytes(), point)) bcSpout.dao.Store.Put(storage.SYSStateSyncPoint.Bytes(), point)
shouldFail := stage == 0x03 // unknown stage shouldFail := stage == 0x03 // unknown stage
checkNewBlockchainErr(t, spountCfg, bcSpout.dao.Store, shouldFail) checkNewBlockchainErr(t, spountCfg, bcSpout.dao.Store, shouldFail)
}) })

267
pkg/core/dao/dao.go
View file

@ -5,18 +5,17 @@ import (
"context" "context"
"encoding/binary" "encoding/binary"
"errors" "errors"
"fmt"
iocore "io" iocore "io"
"sort" "sort"
"github.com/nspcc-dev/neo-go/pkg/core/block" "github.com/nspcc-dev/neo-go/pkg/core/block"
"github.com/nspcc-dev/neo-go/pkg/core/mpt"
"github.com/nspcc-dev/neo-go/pkg/core/state" "github.com/nspcc-dev/neo-go/pkg/core/state"
"github.com/nspcc-dev/neo-go/pkg/core/storage" "github.com/nspcc-dev/neo-go/pkg/core/storage"
"github.com/nspcc-dev/neo-go/pkg/core/transaction" "github.com/nspcc-dev/neo-go/pkg/core/transaction"
"github.com/nspcc-dev/neo-go/pkg/io" "github.com/nspcc-dev/neo-go/pkg/io"
"github.com/nspcc-dev/neo-go/pkg/smartcontract/trigger" "github.com/nspcc-dev/neo-go/pkg/smartcontract/trigger"
"github.com/nspcc-dev/neo-go/pkg/util" "github.com/nspcc-dev/neo-go/pkg/util"
"github.com/nspcc-dev/neo-go/pkg/util/slice"
) )
// HasTransaction errors. // HasTransaction errors.
@ -28,57 +27,21 @@ var (
ErrHasConflicts = errors.New("transaction has conflicts") ErrHasConflicts = errors.New("transaction has conflicts")
) )
// DAO is a data access object.
type DAO interface {
DeleteBlock(h util.Uint256, buf *io.BufBinWriter) error
DeleteContractID(id int32) error
DeleteStorageItem(id int32, key []byte) error
GetAndDecode(entity io.Serializable, key []byte) error
GetAppExecResults(hash util.Uint256, trig trigger.Type) ([]state.AppExecResult, error)
GetBatch() *storage.MemBatch
GetBlock(hash util.Uint256) (*block.Block, error)
GetContractScriptHash(id int32) (util.Uint160, error)
GetCurrentBlockHeight() (uint32, error)
GetCurrentHeaderHeight() (i uint32, h util.Uint256, err error)
GetHeaderHashes() ([]util.Uint256, error)
GetTokenTransferInfo(acc util.Uint160) (*state.TokenTransferInfo, error)
GetTokenTransferLog(acc util.Uint160, start uint64, index uint32, isNEP11 bool) (*state.TokenTransferLog, error)
GetStateSyncPoint() (uint32, error)
GetStateSyncCurrentBlockHeight() (uint32, error)
GetStorageItem(id int32, key []byte) state.StorageItem
GetStorageItems(id int32) ([]state.StorageItemWithKey, error)
GetStorageItemsWithPrefix(id int32, prefix []byte) ([]state.StorageItemWithKey, error)
GetTransaction(hash util.Uint256) (*transaction.Transaction, uint32, error)
GetVersion() (Version, error)
GetWrapped() DAO
HasTransaction(hash util.Uint256) error
Persist() (int, error)
PersistSync() (int, error)
PutContractID(id int32, hash util.Uint160) error
PutCurrentHeader(hashAndIndex []byte) error
PutTokenTransferInfo(acc util.Uint160, bs *state.TokenTransferInfo) error
PutTokenTransferLog(acc util.Uint160, start uint64, index uint32, isNEP11 bool, lg *state.TokenTransferLog) error
PutStateSyncPoint(p uint32) error
PutStateSyncCurrentBlockHeight(h uint32) error
PutStorageItem(id int32, key []byte, si state.StorageItem) error
PutVersion(v Version) error
Seek(id int32, rng storage.SeekRange, f func(k, v []byte) bool)
SeekAsync(ctx context.Context, id int32, rng storage.SeekRange) chan storage.KeyValue
StoreAsBlock(block *block.Block, aer1 *state.AppExecResult, aer2 *state.AppExecResult, buf *io.BufBinWriter) error
StoreAsCurrentBlock(block *block.Block, buf *io.BufBinWriter) error
StoreAsTransaction(tx *transaction.Transaction, index uint32, aer *state.AppExecResult, buf *io.BufBinWriter) error
putTokenTransferInfo(acc util.Uint160, bs *state.TokenTransferInfo, buf *io.BufBinWriter) error
}
// Simple is memCached wrapper around DB, simple DAO implementation. // Simple is memCached wrapper around DB, simple DAO implementation.
type Simple struct { type Simple struct {
Version Version Version Version
Store *storage.MemCachedStore Store *storage.MemCachedStore
keyBuf []byte
dataBuf *io.BufBinWriter
} }
// NewSimple creates new simple dao using provided backend store. // NewSimple creates new simple dao using provided backend store.
func NewSimple(backend storage.Store, stateRootInHeader bool, p2pSigExtensions bool) *Simple { func NewSimple(backend storage.Store, stateRootInHeader bool, p2pSigExtensions bool) *Simple {
st := storage.NewMemCachedStore(backend) st := storage.NewMemCachedStore(backend)
return newSimple(st, stateRootInHeader, p2pSigExtensions)
}
func newSimple(st *storage.MemCachedStore, stateRootInHeader bool, p2pSigExtensions bool) *Simple {
return &Simple{ return &Simple{
Version: Version{ Version: Version{
StoragePrefix: storage.STStorage, StoragePrefix: storage.STStorage,
@ -96,12 +59,31 @@ func (dao *Simple) GetBatch() *storage.MemBatch {
// GetWrapped returns new DAO instance with another layer of wrapped // GetWrapped returns new DAO instance with another layer of wrapped
// MemCachedStore around the current DAO Store. // MemCachedStore around the current DAO Store.
func (dao *Simple) GetWrapped() DAO { func (dao *Simple) GetWrapped() *Simple {
d := NewSimple(dao.Store, dao.Version.StateRootInHeader, dao.Version.P2PSigExtensions) d := NewSimple(dao.Store, dao.Version.StateRootInHeader, dao.Version.P2PSigExtensions)
d.Version = dao.Version d.Version = dao.Version
return d return d
} }
// GetPrivate returns new DAO instance with another layer of private
// MemCachedStore around the current DAO Store.
func (dao *Simple) GetPrivate() *Simple {
st := storage.NewPrivateMemCachedStore(dao.Store)
d := newSimple(st, dao.Version.StateRootInHeader, dao.Version.P2PSigExtensions)
d.Version = dao.Version
if dao.keyBuf != nil { // This one is private.
d.keyBuf = dao.keyBuf // Thus we can reuse its buffer.
} else {
d.keyBuf = make([]byte, 0, 1+4+storage.MaxStorageKeyLen) // Prefix, uint32, key.
}
if dao.dataBuf != nil { // This one is private.
d.dataBuf = dao.dataBuf // Thus we can reuse its buffer.
} else {
d.dataBuf = io.NewBufBinWriter()
}
return d
}
// GetAndDecode performs get operation and decoding with serializable structures. // GetAndDecode performs get operation and decoding with serializable structures.
func (dao *Simple) GetAndDecode(entity io.Serializable, key []byte) error { func (dao *Simple) GetAndDecode(entity io.Serializable, key []byte) error {
entityBytes, err := dao.Store.Get(key) entityBytes, err := dao.Store.Get(key)
@ -113,41 +95,37 @@ func (dao *Simple) GetAndDecode(entity io.Serializable, key []byte) error {
return reader.Err return reader.Err
} }
// Put performs put operation with serializable structures.
func (dao *Simple) Put(entity io.Serializable, key []byte) error {
return dao.putWithBuffer(entity, key, io.NewBufBinWriter())
}
// putWithBuffer performs put operation using buf as a pre-allocated buffer for serialization. // putWithBuffer performs put operation using buf as a pre-allocated buffer for serialization.
func (dao *Simple) putWithBuffer(entity io.Serializable, key []byte, buf *io.BufBinWriter) error { func (dao *Simple) putWithBuffer(entity io.Serializable, key []byte, buf *io.BufBinWriter) error {
entity.EncodeBinary(buf.BinWriter) entity.EncodeBinary(buf.BinWriter)
if buf.Err != nil { if buf.Err != nil {
return buf.Err return buf.Err
} }
return dao.Store.Put(key, buf.Bytes()) dao.Store.Put(key, buf.Bytes())
return nil
} }
func makeContractIDKey(id int32) []byte { func (dao *Simple) makeContractIDKey(id int32) []byte {
key := make([]byte, 5) key := dao.getKeyBuf(5)
key[0] = byte(storage.STContractID) key[0] = byte(storage.STContractID)
binary.LittleEndian.PutUint32(key[1:], uint32(id)) binary.LittleEndian.PutUint32(key[1:], uint32(id))
return key return key
} }
// DeleteContractID deletes contract's id to hash mapping. // DeleteContractID deletes contract's id to hash mapping.
func (dao *Simple) DeleteContractID(id int32) error { func (dao *Simple) DeleteContractID(id int32) {
return dao.Store.Delete(makeContractIDKey(id)) dao.Store.Delete(dao.makeContractIDKey(id))
} }
// PutContractID adds a mapping from contract's ID to its hash. // PutContractID adds a mapping from contract's ID to its hash.
func (dao *Simple) PutContractID(id int32, hash util.Uint160) error { func (dao *Simple) PutContractID(id int32, hash util.Uint160) {
return dao.Store.Put(makeContractIDKey(id), hash.BytesBE()) dao.Store.Put(dao.makeContractIDKey(id), hash.BytesBE())
} }
// GetContractScriptHash retrieves contract's hash given its ID. // GetContractScriptHash retrieves contract's hash given its ID.
func (dao *Simple) GetContractScriptHash(id int32) (util.Uint160, error) { func (dao *Simple) GetContractScriptHash(id int32) (util.Uint160, error) {
var data = new(util.Uint160) var data = new(util.Uint160)
if err := dao.GetAndDecode(data, makeContractIDKey(id)); err != nil { if err := dao.GetAndDecode(data, dao.makeContractIDKey(id)); err != nil {
return *data, err return *data, err
} }
return *data, nil return *data, nil
@ -155,9 +133,16 @@ func (dao *Simple) GetContractScriptHash(id int32) (util.Uint160, error) {
// -- start NEP-17 transfer info. // -- start NEP-17 transfer info.
func (dao *Simple) makeTTIKey(acc util.Uint160) []byte {
key := dao.getKeyBuf(1 + util.Uint160Size)
key[0] = byte(storage.STTokenTransferInfo)
copy(key[1:], acc.BytesBE())
return key
}
// GetTokenTransferInfo retrieves NEP-17 transfer info from the cache. // GetTokenTransferInfo retrieves NEP-17 transfer info from the cache.
func (dao *Simple) GetTokenTransferInfo(acc util.Uint160) (*state.TokenTransferInfo, error) { func (dao *Simple) GetTokenTransferInfo(acc util.Uint160) (*state.TokenTransferInfo, error) {
key := storage.AppendPrefix(storage.STTokenTransferInfo, acc.BytesBE()) key := dao.makeTTIKey(acc)
bs := state.NewTokenTransferInfo() bs := state.NewTokenTransferInfo()
err := dao.GetAndDecode(bs, key) err := dao.GetAndDecode(bs, key)
if err != nil && err != storage.ErrKeyNotFound { if err != nil && err != storage.ErrKeyNotFound {
@ -168,20 +153,19 @@ func (dao *Simple) GetTokenTransferInfo(acc util.Uint160) (*state.TokenTransferI
// PutTokenTransferInfo saves NEP-17 transfer info in the cache. // PutTokenTransferInfo saves NEP-17 transfer info in the cache.
func (dao *Simple) PutTokenTransferInfo(acc util.Uint160, bs *state.TokenTransferInfo) error { func (dao *Simple) PutTokenTransferInfo(acc util.Uint160, bs *state.TokenTransferInfo) error {
return dao.putTokenTransferInfo(acc, bs, io.NewBufBinWriter()) return dao.putTokenTransferInfo(acc, bs, dao.getDataBuf())
} }
func (dao *Simple) putTokenTransferInfo(acc util.Uint160, bs *state.TokenTransferInfo, buf *io.BufBinWriter) error { func (dao *Simple) putTokenTransferInfo(acc util.Uint160, bs *state.TokenTransferInfo, buf *io.BufBinWriter) error {
key := storage.AppendPrefix(storage.STTokenTransferInfo, acc.BytesBE()) return dao.putWithBuffer(bs, dao.makeTTIKey(acc), buf)
return dao.putWithBuffer(bs, key, buf)
} }
// -- end NEP-17 transfer info. // -- end NEP-17 transfer info.
// -- start transfer log. // -- start transfer log.
func getTokenTransferLogKey(acc util.Uint160, newestTimestamp uint64, index uint32, isNEP11 bool) []byte { func (dao *Simple) getTokenTransferLogKey(acc util.Uint160, newestTimestamp uint64, index uint32, isNEP11 bool) []byte {
key := make([]byte, 1+util.Uint160Size+8+4) key := dao.getKeyBuf(1 + util.Uint160Size + 8 + 4)
if isNEP11 { if isNEP11 {
key[0] = byte(storage.STNEP11Transfers) key[0] = byte(storage.STNEP11Transfers)
} else { } else {
@ -197,7 +181,7 @@ func getTokenTransferLogKey(acc util.Uint160, newestTimestamp uint64, index uint
// the transfer with the newest timestamp up to the oldest transfer. It continues // the transfer with the newest timestamp up to the oldest transfer. It continues
// iteration until false is returned from f. The last non-nil error is returned. // iteration until false is returned from f. The last non-nil error is returned.
func (dao *Simple) SeekNEP17TransferLog(acc util.Uint160, newestTimestamp uint64, f func(*state.NEP17Transfer) (bool, error)) error { func (dao *Simple) SeekNEP17TransferLog(acc util.Uint160, newestTimestamp uint64, f func(*state.NEP17Transfer) (bool, error)) error {
key := getTokenTransferLogKey(acc, newestTimestamp, 0, false) key := dao.getTokenTransferLogKey(acc, newestTimestamp, 0, false)
prefixLen := 1 + util.Uint160Size prefixLen := 1 + util.Uint160Size
var seekErr error var seekErr error
dao.Store.Seek(storage.SeekRange{ dao.Store.Seek(storage.SeekRange{
@ -219,7 +203,7 @@ func (dao *Simple) SeekNEP17TransferLog(acc util.Uint160, newestTimestamp uint64
// the transfer with the newest timestamp up to the oldest transfer. It continues // the transfer with the newest timestamp up to the oldest transfer. It continues
// iteration until false is returned from f. The last non-nil error is returned. // iteration until false is returned from f. The last non-nil error is returned.
func (dao *Simple) SeekNEP11TransferLog(acc util.Uint160, newestTimestamp uint64, f func(*state.NEP11Transfer) (bool, error)) error { func (dao *Simple) SeekNEP11TransferLog(acc util.Uint160, newestTimestamp uint64, f func(*state.NEP11Transfer) (bool, error)) error {
key := getTokenTransferLogKey(acc, newestTimestamp, 0, true) key := dao.getTokenTransferLogKey(acc, newestTimestamp, 0, true)
prefixLen := 1 + util.Uint160Size prefixLen := 1 + util.Uint160Size
var seekErr error var seekErr error
dao.Store.Seek(storage.SeekRange{ dao.Store.Seek(storage.SeekRange{
@ -239,7 +223,7 @@ func (dao *Simple) SeekNEP11TransferLog(acc util.Uint160, newestTimestamp uint64
// GetTokenTransferLog retrieves transfer log from the cache. // GetTokenTransferLog retrieves transfer log from the cache.
func (dao *Simple) GetTokenTransferLog(acc util.Uint160, newestTimestamp uint64, index uint32, isNEP11 bool) (*state.TokenTransferLog, error) { func (dao *Simple) GetTokenTransferLog(acc util.Uint160, newestTimestamp uint64, index uint32, isNEP11 bool) (*state.TokenTransferLog, error) {
key := getTokenTransferLogKey(acc, newestTimestamp, index, isNEP11) key := dao.getTokenTransferLogKey(acc, newestTimestamp, index, isNEP11)
value, err := dao.Store.Get(key) value, err := dao.Store.Get(key)
if err != nil { if err != nil {
if err == storage.ErrKeyNotFound { if err == storage.ErrKeyNotFound {
@ -251,19 +235,26 @@ func (dao *Simple) GetTokenTransferLog(acc util.Uint160, newestTimestamp uint64,
} }
// PutTokenTransferLog saves given transfer log in the cache. // PutTokenTransferLog saves given transfer log in the cache.
func (dao *Simple) PutTokenTransferLog(acc util.Uint160, start uint64, index uint32, isNEP11 bool, lg *state.TokenTransferLog) error { func (dao *Simple) PutTokenTransferLog(acc util.Uint160, start uint64, index uint32, isNEP11 bool, lg *state.TokenTransferLog) {
key := getTokenTransferLogKey(acc, start, index, isNEP11) key := dao.getTokenTransferLogKey(acc, start, index, isNEP11)
return dao.Store.Put(key, lg.Raw) dao.Store.Put(key, lg.Raw)
} }
// -- end transfer log. // -- end transfer log.
// -- start notification event. // -- start notification event.
func (dao *Simple) makeExecutableKey(hash util.Uint256) []byte {
key := dao.getKeyBuf(1 + util.Uint256Size)
key[0] = byte(storage.DataExecutable)
copy(key[1:], hash.BytesBE())
return key
}
// GetAppExecResults gets application execution results with the specified trigger from the // GetAppExecResults gets application execution results with the specified trigger from the
// given store. // given store.
func (dao *Simple) GetAppExecResults(hash util.Uint256, trig trigger.Type) ([]state.AppExecResult, error) { func (dao *Simple) GetAppExecResults(hash util.Uint256, trig trigger.Type) ([]state.AppExecResult, error) {
key := storage.AppendPrefix(storage.DataExecutable, hash.BytesBE()) key := dao.makeExecutableKey(hash)
bs, err := dao.Store.Get(key) bs, err := dao.Store.Get(key)
if err != nil { if err != nil {
return nil, err return nil, err
@ -306,7 +297,7 @@ func (dao *Simple) GetAppExecResults(hash util.Uint256, trig trigger.Type) ([]st
// GetStorageItem returns StorageItem if it exists in the given store. // GetStorageItem returns StorageItem if it exists in the given store.
func (dao *Simple) GetStorageItem(id int32, key []byte) state.StorageItem { func (dao *Simple) GetStorageItem(id int32, key []byte) state.StorageItem {
b, err := dao.Store.Get(makeStorageItemKey(dao.Version.StoragePrefix, id, key)) b, err := dao.Store.Get(dao.makeStorageItemKey(id, key))
if err != nil { if err != nil {
return nil return nil
} }
@ -315,16 +306,16 @@ func (dao *Simple) GetStorageItem(id int32, key []byte) state.StorageItem {
// PutStorageItem puts given StorageItem for given id with given // PutStorageItem puts given StorageItem for given id with given
// key into the given store. // key into the given store.
func (dao *Simple) PutStorageItem(id int32, key []byte, si state.StorageItem) error { func (dao *Simple) PutStorageItem(id int32, key []byte, si state.StorageItem) {
stKey := makeStorageItemKey(dao.Version.StoragePrefix, id, key) stKey := dao.makeStorageItemKey(id, key)
return dao.Store.Put(stKey, si) dao.Store.Put(stKey, si)
} }
// DeleteStorageItem drops storage item for the given id with the // DeleteStorageItem drops storage item for the given id with the
// given key from the store. // given key from the store.
func (dao *Simple) DeleteStorageItem(id int32, key []byte) error { func (dao *Simple) DeleteStorageItem(id int32, key []byte) {
stKey := makeStorageItemKey(dao.Version.StoragePrefix, id, key) stKey := dao.makeStorageItemKey(id, key)
return dao.Store.Delete(stKey) dao.Store.Delete(stKey)
} }
// GetStorageItems returns all storage items for a given id. // GetStorageItems returns all storage items for a given id.
@ -354,7 +345,7 @@ func (dao *Simple) GetStorageItemsWithPrefix(id int32, prefix []byte) ([]state.S
// starting from the point specified). If key or value is to be used outside of f, they // starting from the point specified). If key or value is to be used outside of f, they
// may not be copied. Seek continues iterating until false is returned from f. // may not be copied. Seek continues iterating until false is returned from f.
func (dao *Simple) Seek(id int32, rng storage.SeekRange, f func(k, v []byte) bool) { func (dao *Simple) Seek(id int32, rng storage.SeekRange, f func(k, v []byte) bool) {
rng.Prefix = makeStorageItemKey(dao.Version.StoragePrefix, id, rng.Prefix) rng.Prefix = slice.Copy(dao.makeStorageItemKey(id, rng.Prefix)) // f() can use dao too.
dao.Store.Seek(rng, func(k, v []byte) bool { dao.Store.Seek(rng, func(k, v []byte) bool {
return f(k[len(rng.Prefix):], v) return f(k[len(rng.Prefix):], v)
}) })
@ -364,15 +355,15 @@ func (dao *Simple) Seek(id int32, rng storage.SeekRange, f func(k, v []byte) boo
// starting from the point specified) to a channel and returns the channel. // starting from the point specified) to a channel and returns the channel.
// Resulting keys and values may not be copied. // Resulting keys and values may not be copied.
func (dao *Simple) SeekAsync(ctx context.Context, id int32, rng storage.SeekRange) chan storage.KeyValue { func (dao *Simple) SeekAsync(ctx context.Context, id int32, rng storage.SeekRange) chan storage.KeyValue {
rng.Prefix = makeStorageItemKey(dao.Version.StoragePrefix, id, rng.Prefix) rng.Prefix = slice.Copy(dao.makeStorageItemKey(id, rng.Prefix))
return dao.Store.SeekAsync(ctx, rng, true) return dao.Store.SeekAsync(ctx, rng, true)
} }
// makeStorageItemKey returns a key used to store StorageItem in the DB. // makeStorageItemKey returns a key used to store StorageItem in the DB.
func makeStorageItemKey(prefix storage.KeyPrefix, id int32, key []byte) []byte { func (dao *Simple) makeStorageItemKey(id int32, key []byte) []byte {
// 1 for prefix + 4 for Uint32 + len(key) for key // 1 for prefix + 4 for Uint32 + len(key) for key
buf := make([]byte, 5+len(key)) buf := dao.getKeyBuf(5 + len(key))
buf[0] = byte(prefix) buf[0] = byte(dao.Version.StoragePrefix)
binary.LittleEndian.PutUint32(buf[1:], uint32(id)) binary.LittleEndian.PutUint32(buf[1:], uint32(id))
copy(buf[5:], key) copy(buf[5:], key)
return buf return buf
@ -384,7 +375,7 @@ func makeStorageItemKey(prefix storage.KeyPrefix, id int32, key []byte) []byte {
// GetBlock returns Block by the given hash if it exists in the store. // GetBlock returns Block by the given hash if it exists in the store.
func (dao *Simple) GetBlock(hash util.Uint256) (*block.Block, error) { func (dao *Simple) GetBlock(hash util.Uint256) (*block.Block, error) {
key := storage.AppendPrefix(storage.DataExecutable, hash.BytesBE()) key := dao.makeExecutableKey(hash)
b, err := dao.Store.Get(key) b, err := dao.Store.Get(key)
if err != nil { if err != nil {
return nil, err return nil, err
@ -553,7 +544,7 @@ func (dao *Simple) GetHeaderHashes() ([]util.Uint256, error) {
// GetTransaction returns Transaction and its height by the given hash // GetTransaction returns Transaction and its height by the given hash
// if it exists in the store. It does not return dummy transactions. // if it exists in the store. It does not return dummy transactions.
func (dao *Simple) GetTransaction(hash util.Uint256) (*transaction.Transaction, uint32, error) { func (dao *Simple) GetTransaction(hash util.Uint256) (*transaction.Transaction, uint32, error) {
key := storage.AppendPrefix(storage.DataExecutable, hash.BytesBE()) key := dao.makeExecutableKey(hash)
b, err := dao.Store.Get(key) b, err := dao.Store.Get(key)
if err != nil { if err != nil {
return nil, 0, err return nil, 0, err
@ -582,28 +573,28 @@ func (dao *Simple) GetTransaction(hash util.Uint256) (*transaction.Transaction,
} }
// PutVersion stores the given version in the underlying store. // PutVersion stores the given version in the underlying store.
func (dao *Simple) PutVersion(v Version) error { func (dao *Simple) PutVersion(v Version) {
dao.Version = v dao.Version = v
return dao.Store.Put(storage.SYSVersion.Bytes(), v.Bytes()) dao.Store.Put(storage.SYSVersion.Bytes(), v.Bytes())
} }
// PutCurrentHeader stores current header. // PutCurrentHeader stores current header.
func (dao *Simple) PutCurrentHeader(hashAndIndex []byte) error { func (dao *Simple) PutCurrentHeader(hashAndIndex []byte) {
return dao.Store.Put(storage.SYSCurrentHeader.Bytes(), hashAndIndex) dao.Store.Put(storage.SYSCurrentHeader.Bytes(), hashAndIndex)
} }
// PutStateSyncPoint stores current state synchronisation point P. // PutStateSyncPoint stores current state synchronisation point P.
func (dao *Simple) PutStateSyncPoint(p uint32) error { func (dao *Simple) PutStateSyncPoint(p uint32) {
buf := make([]byte, 4) buf := dao.getKeyBuf(4) // It's very small, no point in using BufBinWriter.
binary.LittleEndian.PutUint32(buf, p) binary.LittleEndian.PutUint32(buf, p)
return dao.Store.Put(storage.SYSStateSyncPoint.Bytes(), buf) dao.Store.Put(storage.SYSStateSyncPoint.Bytes(), buf)
} }
// PutStateSyncCurrentBlockHeight stores current block height during state synchronisation process. // PutStateSyncCurrentBlockHeight stores current block height during state synchronisation process.
func (dao *Simple) PutStateSyncCurrentBlockHeight(h uint32) error { func (dao *Simple) PutStateSyncCurrentBlockHeight(h uint32) {
buf := make([]byte, 4) buf := dao.getKeyBuf(4) // It's very small, no point in using BufBinWriter.
binary.LittleEndian.PutUint32(buf, h) binary.LittleEndian.PutUint32(buf, h)
return dao.Store.Put(storage.SYSStateSyncCurrentBlockHeight.Bytes(), buf) dao.Store.Put(storage.SYSStateSyncCurrentBlockHeight.Bytes(), buf)
} }
// read2000Uint256Hashes attempts to read 2000 Uint256 hashes from // read2000Uint256Hashes attempts to read 2000 Uint256 hashes from
@ -623,7 +614,7 @@ func read2000Uint256Hashes(b []byte) ([]util.Uint256, error) {
// Transaction hash. It returns an error in case if transaction is in chain // Transaction hash. It returns an error in case if transaction is in chain
// or in the list of conflicting transactions. // or in the list of conflicting transactions.
func (dao *Simple) HasTransaction(hash util.Uint256) error { func (dao *Simple) HasTransaction(hash util.Uint256) error {
key := storage.AppendPrefix(storage.DataExecutable, hash.BytesBE()) key := dao.makeExecutableKey(hash)
bytes, err := dao.Store.Get(key) bytes, err := dao.Store.Get(key)
if err != nil { if err != nil {
return nil return nil
@ -640,13 +631,11 @@ func (dao *Simple) HasTransaction(hash util.Uint256) error {
// StoreAsBlock stores given block as DataBlock. It can reuse given buffer for // StoreAsBlock stores given block as DataBlock. It can reuse given buffer for
// the purpose of value serialization. // the purpose of value serialization.
func (dao *Simple) StoreAsBlock(block *block.Block, aer1 *state.AppExecResult, aer2 *state.AppExecResult, buf *io.BufBinWriter) error { func (dao *Simple) StoreAsBlock(block *block.Block, aer1 *state.AppExecResult, aer2 *state.AppExecResult) error {
var ( var (
key = storage.AppendPrefix(storage.DataExecutable, block.Hash().BytesBE()) key = dao.makeExecutableKey(block.Hash())
buf = dao.getDataBuf()
) )
if buf == nil {
buf = io.NewBufBinWriter()
}
buf.WriteB(storage.ExecBlock) buf.WriteB(storage.ExecBlock)
b, err := block.Trim() b, err := block.Trim()
if err != nil { if err != nil {
@ -662,15 +651,14 @@ func (dao *Simple) StoreAsBlock(block *block.Block, aer1 *state.AppExecResult, a
if buf.Err != nil { if buf.Err != nil {
return buf.Err return buf.Err
} }
return dao.Store.Put(key, buf.Bytes()) dao.Store.Put(key, buf.Bytes())
return nil
} }
// DeleteBlock removes block from dao. // DeleteBlock removes block from dao. It's not atomic, so make sure you're
func (dao *Simple) DeleteBlock(h util.Uint256, w *io.BufBinWriter) error { // using private MemCached instance here.
batch := dao.Store.Batch() func (dao *Simple) DeleteBlock(h util.Uint256) error {
key := make([]byte, util.Uint256Size+1) key := dao.makeExecutableKey(h)
key[0] = byte(storage.DataExecutable)
copy(key[1:], h.BytesBE())
bs, err := dao.Store.Get(key) bs, err := dao.Store.Get(key)
if err != nil { if err != nil {
return err return err
@ -685,53 +673,48 @@ func (dao *Simple) DeleteBlock(h util.Uint256, w *io.BufBinWriter) error {
return err return err
} }
if w == nil { w := dao.getDataBuf()
w = io.NewBufBinWriter()
}
w.WriteB(storage.ExecBlock) w.WriteB(storage.ExecBlock)
b.Header.EncodeBinary(w.BinWriter) b.Header.EncodeBinary(w.BinWriter)
w.BinWriter.WriteB(0) w.BinWriter.WriteB(0)
if w.Err != nil { if w.Err != nil {
return w.Err return w.Err
} }
batch.Put(key, w.Bytes()) dao.Store.Put(key, w.Bytes())
for _, tx := range b.Transactions { for _, tx := range b.Transactions {
copy(key[1:], tx.Hash().BytesBE()) copy(key[1:], tx.Hash().BytesBE())
batch.Delete(key) dao.Store.Delete(key)
if dao.Version.P2PSigExtensions { if dao.Version.P2PSigExtensions {
for _, attr := range tx.GetAttributes(transaction.ConflictsT) { for _, attr := range tx.GetAttributes(transaction.ConflictsT) {
hash := attr.Value.(*transaction.Conflicts).Hash hash := attr.Value.(*transaction.Conflicts).Hash
copy(key[1:], hash.BytesBE()) copy(key[1:], hash.BytesBE())
batch.Delete(key) dao.Store.Delete(key)
} }
} }
} }
return dao.Store.PutBatch(batch) return nil
} }
// StoreAsCurrentBlock stores a hash of the given block with prefix // StoreAsCurrentBlock stores a hash of the given block with prefix
// SYSCurrentBlock. It can reuse given buffer for the purpose of value // SYSCurrentBlock. It can reuse given buffer for the purpose of value
// serialization. // serialization.
func (dao *Simple) StoreAsCurrentBlock(block *block.Block, buf *io.BufBinWriter) error { func (dao *Simple) StoreAsCurrentBlock(block *block.Block) {
if buf == nil { buf := dao.getDataBuf()
buf = io.NewBufBinWriter()
}
h := block.Hash() h := block.Hash()
h.EncodeBinary(buf.BinWriter) h.EncodeBinary(buf.BinWriter)
buf.WriteU32LE(block.Index) buf.WriteU32LE(block.Index)
return dao.Store.Put(storage.SYSCurrentBlock.Bytes(), buf.Bytes()) dao.Store.Put(storage.SYSCurrentBlock.Bytes(), buf.Bytes())
} }
// StoreAsTransaction stores given TX as DataTransaction. It also stores transactions // StoreAsTransaction stores given TX as DataTransaction. It also stores transactions
// given tx has conflicts with as DataTransaction with dummy version. It can reuse given // given tx has conflicts with as DataTransaction with dummy version. It can reuse given
// buffer for the purpose of value serialization. // buffer for the purpose of value serialization.
func (dao *Simple) StoreAsTransaction(tx *transaction.Transaction, index uint32, aer *state.AppExecResult, buf *io.BufBinWriter) error { func (dao *Simple) StoreAsTransaction(tx *transaction.Transaction, index uint32, aer *state.AppExecResult) error {
key := storage.AppendPrefix(storage.DataExecutable, tx.Hash().BytesBE()) key := dao.makeExecutableKey(tx.Hash())
if buf == nil { buf := dao.getDataBuf()
buf = io.NewBufBinWriter()
}
buf.WriteB(storage.ExecTransaction) buf.WriteB(storage.ExecTransaction)
buf.WriteU32LE(index) buf.WriteU32LE(index)
tx.EncodeBinary(buf.BinWriter) tx.EncodeBinary(buf.BinWriter)
@ -741,10 +724,7 @@ func (dao *Simple) StoreAsTransaction(tx *transaction.Transaction, index uint32,
if buf.Err != nil { if buf.Err != nil {
return buf.Err return buf.Err
} }
err := dao.Store.Put(key, buf.Bytes()) dao.Store.Put(key, buf.Bytes())
if err != nil {
return err
}
if dao.Version.P2PSigExtensions { if dao.Version.P2PSigExtensions {
var value []byte var value []byte
for _, attr := range tx.GetAttributes(transaction.ConflictsT) { for _, attr := range tx.GetAttributes(transaction.ConflictsT) {
@ -757,15 +737,27 @@ func (dao *Simple) StoreAsTransaction(tx *transaction.Transaction, index uint32,
buf.BinWriter.WriteB(transaction.DummyVersion) buf.BinWriter.WriteB(transaction.DummyVersion)
value = buf.Bytes() value = buf.Bytes()
} }
err = dao.Store.Put(key, value) dao.Store.Put(key, value)
if err != nil {
return fmt.Errorf("failed to store conflicting transaction %s for transaction %s: %w", hash.StringLE(), tx.Hash().StringLE(), err)
}
} }
} }
return nil return nil
} }
func (dao *Simple) getKeyBuf(len int) []byte {
if dao.keyBuf != nil { // Private DAO.
return dao.keyBuf[:len] // Should have enough capacity.
}
return make([]byte, len)
}
func (dao *Simple) getDataBuf() *io.BufBinWriter {
if dao.dataBuf != nil {
dao.dataBuf.Reset()
return dao.dataBuf
}
return io.NewBufBinWriter()
}
// Persist flushes all the changes made into the (supposedly) persistent // Persist flushes all the changes made into the (supposedly) persistent
// underlying store. It doesn't block accesses to DAO from other threads. // underlying store. It doesn't block accesses to DAO from other threads.
func (dao *Simple) Persist() (int, error) { func (dao *Simple) Persist() (int, error) {
@ -778,12 +770,3 @@ func (dao *Simple) Persist() (int, error) {
func (dao *Simple) PersistSync() (int, error) { func (dao *Simple) PersistSync() (int, error) {
return dao.Store.PersistSync() return dao.Store.PersistSync()
} }
// GetMPTBatch storage changes to be applied to MPT.
func (dao *Simple) GetMPTBatch() mpt.Batch {
var b mpt.Batch
dao.Store.MemoryStore.SeekAll([]byte{byte(dao.Version.StoragePrefix)}, func(k, v []byte) {
b.Add(k[1:], v)
})
return b
}

45
pkg/core/dao/dao_test.go
View file

@ -22,11 +22,10 @@ func TestPutGetAndDecode(t *testing.T) {
dao := NewSimple(storage.NewMemoryStore(), false, false) dao := NewSimple(storage.NewMemoryStore(), false, false)
serializable := &TestSerializable{field: random.String(4)} serializable := &TestSerializable{field: random.String(4)}
hash := []byte{1} hash := []byte{1}
err := dao.Put(serializable, hash) require.NoError(t, dao.putWithBuffer(serializable, hash, io.NewBufBinWriter()))
require.NoError(t, err)
gotAndDecoded := &TestSerializable{} gotAndDecoded := &TestSerializable{}
err = dao.GetAndDecode(gotAndDecoded, hash) err := dao.GetAndDecode(gotAndDecoded, hash)
require.NoError(t, err) require.NoError(t, err)
} }
@ -48,8 +47,7 @@ func TestPutGetStorageItem(t *testing.T) {
id := int32(random.Int(0, 1024)) id := int32(random.Int(0, 1024))
key := []byte{0} key := []byte{0}
storageItem := state.StorageItem{} storageItem := state.StorageItem{}
err := dao.PutStorageItem(id, key, storageItem) dao.PutStorageItem(id, key, storageItem)
require.NoError(t, err)
gotStorageItem := dao.GetStorageItem(id, key) gotStorageItem := dao.GetStorageItem(id, key)
require.Equal(t, storageItem, gotStorageItem) require.Equal(t, storageItem, gotStorageItem)
} }
@ -59,10 +57,8 @@ func TestDeleteStorageItem(t *testing.T) {
id := int32(random.Int(0, 1024)) id := int32(random.Int(0, 1024))
key := []byte{0} key := []byte{0}
storageItem := state.StorageItem{} storageItem := state.StorageItem{}
err := dao.PutStorageItem(id, key, storageItem) dao.PutStorageItem(id, key, storageItem)
require.NoError(t, err) dao.DeleteStorageItem(id, key)
err = dao.DeleteStorageItem(id, key)
require.NoError(t, err)
gotStorageItem := dao.GetStorageItem(id, key) gotStorageItem := dao.GetStorageItem(id, key)
require.Nil(t, gotStorageItem) require.Nil(t, gotStorageItem)
} }
@ -102,7 +98,7 @@ func TestPutGetBlock(t *testing.T) {
Stack: []stackitem.Item{}, Stack: []stackitem.Item{},
}, },
} }
err := dao.StoreAsBlock(b, appExecResult1, appExecResult2, nil) err := dao.StoreAsBlock(b, appExecResult1, appExecResult2)
require.NoError(t, err) require.NoError(t, err)
gotBlock, err := dao.GetBlock(hash) gotBlock, err := dao.GetBlock(hash)
require.NoError(t, err) require.NoError(t, err)
@ -129,22 +125,21 @@ func TestGetVersion(t *testing.T) {
StateRootInHeader: true, StateRootInHeader: true,
Value: "testVersion", Value: "testVersion",
} }
err := dao.PutVersion(expected) dao.PutVersion(expected)
require.NoError(t, err)
actual, err := dao.GetVersion() actual, err := dao.GetVersion()
require.NoError(t, err) require.NoError(t, err)
require.Equal(t, expected, actual) require.Equal(t, expected, actual)
t.Run("invalid", func(t *testing.T) { t.Run("invalid", func(t *testing.T) {
dao := NewSimple(storage.NewMemoryStore(), false, false) dao := NewSimple(storage.NewMemoryStore(), false, false)
require.NoError(t, dao.Store.Put(storage.SYSVersion.Bytes(), []byte("0.1.2\x00x"))) dao.Store.Put(storage.SYSVersion.Bytes(), []byte("0.1.2\x00x"))
_, err := dao.GetVersion() _, err := dao.GetVersion()
require.Error(t, err) require.Error(t, err)
}) })
t.Run("old format", func(t *testing.T) { t.Run("old format", func(t *testing.T) {
dao := NewSimple(storage.NewMemoryStore(), false, false) dao := NewSimple(storage.NewMemoryStore(), false, false)
require.NoError(t, dao.Store.Put(storage.SYSVersion.Bytes(), []byte("0.1.2"))) dao.Store.Put(storage.SYSVersion.Bytes(), []byte("0.1.2"))
version, err := dao.GetVersion() version, err := dao.GetVersion()
require.NoError(t, err) require.NoError(t, err)
@ -169,8 +164,7 @@ func TestGetCurrentHeaderHeight_Store(t *testing.T) {
}, },
}, },
} }
err := dao.StoreAsCurrentBlock(b, nil) dao.StoreAsCurrentBlock(b)
require.NoError(t, err)
height, err := dao.GetCurrentBlockHeight() height, err := dao.GetCurrentBlockHeight()
require.NoError(t, err) require.NoError(t, err)
require.Equal(t, uint32(0), height) require.Equal(t, uint32(0), height)
@ -191,7 +185,7 @@ func TestStoreAsTransaction(t *testing.T) {
Stack: []stackitem.Item{}, Stack: []stackitem.Item{},
}, },
} }
err := dao.StoreAsTransaction(tx, 0, aer, nil) err := dao.StoreAsTransaction(tx, 0, aer)
require.NoError(t, err) require.NoError(t, err)
err = dao.HasTransaction(hash) err = dao.HasTransaction(hash)
require.NotNil(t, err) require.NotNil(t, err)
@ -222,7 +216,7 @@ func TestStoreAsTransaction(t *testing.T) {
Stack: []stackitem.Item{}, Stack: []stackitem.Item{},
}, },
} }
err := dao.StoreAsTransaction(tx, 0, aer, nil) err := dao.StoreAsTransaction(tx, 0, aer)
require.NoError(t, err) require.NoError(t, err)
err = dao.HasTransaction(hash) err = dao.HasTransaction(hash)
require.True(t, errors.Is(err, ErrAlreadyExists)) require.True(t, errors.Is(err, ErrAlreadyExists))
@ -271,7 +265,7 @@ func BenchmarkStoreAsTransaction(b *testing.B) {
b.ResetTimer() b.ResetTimer()
b.ReportAllocs() b.ReportAllocs()
for n := 0; n < b.N; n++ { for n := 0; n < b.N; n++ {
err := dao.StoreAsTransaction(tx, 1, aer, nil) err := dao.StoreAsTransaction(tx, 1, aer)
if err != nil { if err != nil {
b.FailNow() b.FailNow()
} }
@ -281,18 +275,21 @@ func BenchmarkStoreAsTransaction(b *testing.B) {
func TestMakeStorageItemKey(t *testing.T) { func TestMakeStorageItemKey(t *testing.T) {
var id int32 = 5 var id int32 = 5
dao := NewSimple(storage.NewMemoryStore(), true, false)
expected := []byte{byte(storage.STStorage), 0, 0, 0, 0, 1, 2, 3} expected := []byte{byte(storage.STStorage), 0, 0, 0, 0, 1, 2, 3}
binary.LittleEndian.PutUint32(expected[1:5], uint32(id)) binary.LittleEndian.PutUint32(expected[1:5], uint32(id))
actual := makeStorageItemKey(storage.STStorage, id, []byte{1, 2, 3}) actual := dao.makeStorageItemKey(id, []byte{1, 2, 3})
require.Equal(t, expected, actual) require.Equal(t, expected, actual)
expected = expected[0:5] expected = expected[0:5]
actual = makeStorageItemKey(storage.STStorage, id, nil) actual = dao.makeStorageItemKey(id, nil)
require.Equal(t, expected, actual) require.Equal(t, expected, actual)
expected = []byte{byte(storage.STTempStorage), 0, 0, 0, 0, 1, 2, 3} expected = []byte{byte(storage.STTempStorage), 0, 0, 0, 0, 1, 2, 3}
binary.LittleEndian.PutUint32(expected[1:5], uint32(id)) binary.LittleEndian.PutUint32(expected[1:5], uint32(id))
actual = makeStorageItemKey(storage.STTempStorage, id, []byte{1, 2, 3}) dao.Version.StoragePrefix = storage.STTempStorage
actual = dao.makeStorageItemKey(id, []byte{1, 2, 3})
require.Equal(t, expected, actual) require.Equal(t, expected, actual)
} }
@ -305,7 +302,7 @@ func TestPutGetStateSyncPoint(t *testing.T) {
// non-empty store // non-empty store
var expected uint32 = 5 var expected uint32 = 5
require.NoError(t, dao.PutStateSyncPoint(expected)) dao.PutStateSyncPoint(expected)
actual, err := dao.GetStateSyncPoint() actual, err := dao.GetStateSyncPoint()
require.NoError(t, err) require.NoError(t, err)
require.Equal(t, expected, actual) require.Equal(t, expected, actual)
@ -320,7 +317,7 @@ func TestPutGetStateSyncCurrentBlockHeight(t *testing.T) {
// non-empty store // non-empty store
var expected uint32 = 5 var expected uint32 = 5
require.NoError(t, dao.PutStateSyncCurrentBlockHeight(expected)) dao.PutStateSyncCurrentBlockHeight(expected)
actual, err := dao.GetStateSyncCurrentBlockHeight() actual, err := dao.GetStateSyncCurrentBlockHeight()
require.NoError(t, err) require.NoError(t, err)
require.Equal(t, expected, actual) require.Equal(t, expected, actual)

10
pkg/core/interop/context.go
View file

@ -54,24 +54,24 @@ type Context struct {
Block *block.Block Block *block.Block
NonceData [16]byte NonceData [16]byte
Tx *transaction.Transaction Tx *transaction.Transaction
DAO dao.DAO DAO *dao.Simple
Notifications []state.NotificationEvent Notifications []state.NotificationEvent
Log *zap.Logger Log *zap.Logger
VM *vm.VM VM *vm.VM
Functions []Function Functions []Function
Invocations map[util.Uint160]int Invocations map[util.Uint160]int
cancelFuncs []context.CancelFunc cancelFuncs []context.CancelFunc
getContract func(dao.DAO, util.Uint160) (*state.Contract, error) getContract func(*dao.Simple, util.Uint160) (*state.Contract, error)
baseExecFee int64 baseExecFee int64
signers []transaction.Signer signers []transaction.Signer
} }
// NewContext returns new interop context. // NewContext returns new interop context.
func NewContext(trigger trigger.Type, bc Ledger, d dao.DAO, func NewContext(trigger trigger.Type, bc Ledger, d *dao.Simple,
getContract func(dao.DAO, util.Uint160) (*state.Contract, error), natives []Contract, getContract func(*dao.Simple, util.Uint160) (*state.Contract, error), natives []Contract,
block *block.Block, tx *transaction.Transaction, log *zap.Logger) *Context { block *block.Block, tx *transaction.Transaction, log *zap.Logger) *Context {
baseExecFee := int64(DefaultBaseExecFee) baseExecFee := int64(DefaultBaseExecFee)
dao := d.GetWrapped() dao := d.GetPrivate()
if bc != nil && (block == nil || block.Index != 0) { if bc != nil && (block == nil || block.Index != 0) {
baseExecFee = bc.GetBaseExecFee() baseExecFee = bc.GetBaseExecFee()

2
pkg/core/interop/contract/call.go
View file

@ -18,7 +18,7 @@ import (
) )
type policyChecker interface { type policyChecker interface {
IsBlockedInternal(dao.DAO, util.Uint160) bool IsBlockedInternal(*dao.Simple, util.Uint160) bool
} }
// LoadToken calls method specified by token id. // LoadToken calls method specified by token id.

6
pkg/core/interop_system.go
View file

@ -59,7 +59,8 @@ func storageDelete(ic *interop.Context) error {
return errors.New("StorageContext is read only") return errors.New("StorageContext is read only")
} }
key := ic.VM.Estack().Pop().Bytes() key := ic.VM.Estack().Pop().Bytes()
return ic.DAO.DeleteStorageItem(stc.ID, key) ic.DAO.DeleteStorageItem(stc.ID, key)
return nil
} }
// storageGet returns stored key-value pair. // storageGet returns stored key-value pair.
@ -128,7 +129,8 @@ func putWithContext(ic *interop.Context, stc *StorageContext, key []byte, value
if !ic.VM.AddGas(int64(sizeInc) * ic.Chain.GetStoragePrice()) { if !ic.VM.AddGas(int64(sizeInc) * ic.Chain.GetStoragePrice()) {
return errGasLimitExceeded return errGasLimitExceeded
} }
return ic.DAO.PutStorageItem(stc.ID, key, value) ic.DAO.PutStorageItem(stc.ID, key, value)
return nil
} }
// storagePut puts key-value pair into the storage. // storagePut puts key-value pair into the storage.

11
pkg/core/interop_system_test.go
View file

@ -362,8 +362,8 @@ func BenchmarkStorageFind(b *testing.B) {
items["abc"+random.String(10)] = random.Bytes(10) items["abc"+random.String(10)] = random.Bytes(10)
} }
for k, v := range items { for k, v := range items {
require.NoError(b, context.DAO.PutStorageItem(contractState.ID, []byte(k), v)) context.DAO.PutStorageItem(contractState.ID, []byte(k), v)
require.NoError(b, context.DAO.PutStorageItem(contractState.ID+1, []byte(k), v)) context.DAO.PutStorageItem(contractState.ID+1, []byte(k), v)
} }
changes, err := context.DAO.Persist() changes, err := context.DAO.Persist()
require.NoError(b, err) require.NoError(b, err)
@ -406,8 +406,8 @@ func BenchmarkStorageFindIteratorNext(b *testing.B) {
items["abc"+random.String(10)] = random.Bytes(10) items["abc"+random.String(10)] = random.Bytes(10)
} }
for k, v := range items { for k, v := range items {
require.NoError(b, context.DAO.PutStorageItem(contractState.ID, []byte(k), v)) context.DAO.PutStorageItem(contractState.ID, []byte(k), v)
require.NoError(b, context.DAO.PutStorageItem(contractState.ID+1, []byte(k), v)) context.DAO.PutStorageItem(contractState.ID+1, []byte(k), v)
} }
changes, err := context.DAO.Persist() changes, err := context.DAO.Persist()
require.NoError(b, err) require.NoError(b, err)
@ -487,8 +487,7 @@ func TestStorageFind(t *testing.T) {
id := contractState.ID id := contractState.ID
for i := range skeys { for i := range skeys {
err := context.DAO.PutStorageItem(id, skeys[i], items[i]) context.DAO.PutStorageItem(id, skeys[i], items[i])
require.NoError(t, err)
} }
testFind := func(t *testing.T, prefix []byte, opts int64, expected []stackitem.Item) { testFind := func(t *testing.T, prefix []byte, opts int64, expected []stackitem.Item) {

28
pkg/core/mpt/batch.go
View file

@ -16,23 +16,19 @@ type keyValue struct {
value []byte value []byte
} }
// Add adds key-value pair to batch. // MapToMPTBatch makes a Batch from unordered set of storage changes.
// If there is an item with the specified key, it is replaced. func MapToMPTBatch(m map[string][]byte) Batch {
func (b *Batch) Add(key []byte, value []byte) { var b Batch
path := toNibbles(key)
i := sort.Search(len(b.kv), func(i int) bool { b.kv = make([]keyValue, 0, len(m))
return bytes.Compare(path, b.kv[i].key) <= 0
}) for k, v := range m {
if i == len(b.kv) { b.kv = append(b.kv, keyValue{strToNibbles(k), v}) // Strip storage prefix.
b.kv = append(b.kv, keyValue{path, value})
} else if bytes.Equal(b.kv[i].key, path) {
b.kv[i].value = value
} else {
b.kv = append(b.kv, keyValue{})
copy(b.kv[i+1:], b.kv[i:])
b.kv[i].key = path
b.kv[i].value = value
} }
sort.Slice(b.kv, func(i, j int) bool {
return bytes.Compare(b.kv[i].key, b.kv[j].key) < 0
})
return b
} }
// PutBatch puts batch to trie. // PutBatch puts batch to trie.

28
pkg/core/mpt/batch_test.go
View file

@ -10,12 +10,13 @@ import (
) )
func TestBatchAdd(t *testing.T) { func TestBatchAdd(t *testing.T) {
b := new(Batch) b := MapToMPTBatch(map[string][]byte{
b.Add([]byte{1}, []byte{2}) "a\x01": {2},
b.Add([]byte{2, 16}, []byte{3}) "a\x02\x10": {3},
b.Add([]byte{2, 0}, []byte{4}) "a\x00\x01": {5},
b.Add([]byte{0, 1}, []byte{5}) "a\x02\x00": {6},
b.Add([]byte{2, 0}, []byte{6}) })
expected := []keyValue{ expected := []keyValue{
{[]byte{0, 0, 0, 1}, []byte{5}}, {[]byte{0, 0, 0, 1}, []byte{5}},
{[]byte{0, 1}, []byte{2}}, {[]byte{0, 1}, []byte{2}},
@ -28,7 +29,7 @@ func TestBatchAdd(t *testing.T) {
type pairs = [][2][]byte type pairs = [][2][]byte
func testIncompletePut(t *testing.T, ps pairs, n int, tr1, tr2 *Trie) { func testIncompletePut(t *testing.T, ps pairs, n int, tr1, tr2 *Trie) {
var b Batch var m = make(map[string][]byte)
for i, p := range ps { for i, p := range ps {
if i < n { if i < n {
if p[1] == nil { if p[1] == nil {
@ -43,9 +44,10 @@ func testIncompletePut(t *testing.T, ps pairs, n int, tr1, tr2 *Trie) {
require.Error(t, tr1.Put(p[0], p[1]), "item %d", i) require.Error(t, tr1.Put(p[0], p[1]), "item %d", i)
} }
} }
b.Add(p[0], p[1]) m["a"+string(p[0])] = p[1]
} }
b := MapToMPTBatch(m)
num, err := tr2.PutBatch(b) num, err := tr2.PutBatch(b)
if n == len(ps) { if n == len(ps) {
require.NoError(t, err) require.NoError(t, err)
@ -275,8 +277,8 @@ func TestTrie_PutBatchHash(t *testing.T) {
tr1.Collapse(1) tr1.Collapse(1)
tr2.Collapse(1) tr2.Collapse(1)
key := makeStorageKey(tr1.root.(*BranchNode).Children[2].Hash()) key := makeStorageKey(tr1.root.(*BranchNode).Children[2].Hash())
require.NoError(t, tr1.Store.Delete(key)) tr1.Store.Delete(key)
require.NoError(t, tr2.Store.Delete(key)) tr2.Store.Delete(key)
testIncompletePut(t, ps, 1, tr1, tr2) testIncompletePut(t, ps, 1, tr1, tr2)
}) })
} }
@ -308,8 +310,10 @@ func TestTrie_PutBatchEmpty(t *testing.T) {
// For the sake of coverage. // For the sake of coverage.
func TestTrie_InvalidNodeType(t *testing.T) { func TestTrie_InvalidNodeType(t *testing.T) {
tr := NewTrie(EmptyNode{}, ModeAll, newTestStore()) tr := NewTrie(EmptyNode{}, ModeAll, newTestStore())
var b Batch var b = Batch{kv: []keyValue{{
b.Add([]byte{1}, []byte("value")) key: []byte{0, 1},
value: []byte("value"),
}}}
tr.root = Node(nil) tr.root = Node(nil)
require.Panics(t, func() { _, _ = tr.PutBatch(b) }) require.Panics(t, func() { _, _ = tr.PutBatch(b) })
} }

6
pkg/core/mpt/billet.go
View file

@ -70,7 +70,7 @@ func (b *Billet) RestoreHashNode(path []byte, node Node) error {
panic("invalid storage prefix") panic("invalid storage prefix")
} }
k := append([]byte{byte(b.TempStoragePrefix)}, fromNibbles(path)...) k := append([]byte{byte(b.TempStoragePrefix)}, fromNibbles(path)...)
_ = b.Store.Put(k, leaf.value) b.Store.Put(k, leaf.value)
} }
return nil return nil
} }
@ -194,9 +194,9 @@ func (b *Billet) incrementRefAndStore(h util.Uint256, bs []byte) {
data = append(bs, 1, 0, 0, 0, 0) data = append(bs, 1, 0, 0, 0, 0)
} }
binary.LittleEndian.PutUint32(data[len(data)-4:], uint32(cnt)) binary.LittleEndian.PutUint32(data[len(data)-4:], uint32(cnt))
_ = b.Store.Put(key, data) b.Store.Put(key, data)
} else { } else {
_ = b.Store.Put(key, bs) b.Store.Put(key, bs)
} }
} }

11
pkg/core/mpt/helpers.go
View file

@ -43,6 +43,17 @@ func toNibbles(path []byte) []byte {
return result return result
} }
// strToNibbles mangles path by splitting every byte into 2 containing low- and high- 4-byte part,
// ignoring the first byte (prefix).
func strToNibbles(path string) []byte {
result := make([]byte, (len(path)-1)*2)
for i := 0; i < len(path)-1; i++ {
result[i*2] = path[i+1] >> 4
result[i*2+1] = path[i+1] & 0x0F
}
return result
}
// fromNibbles performs operation opposite to toNibbles and does no path validity checks. // fromNibbles performs operation opposite to toNibbles and does no path validity checks.
func fromNibbles(path []byte) []byte { func fromNibbles(path []byte) []byte {
result := make([]byte, len(path)/2) result := make([]byte, len(path)/2)

3
pkg/core/mpt/proof.go
View file

@ -69,8 +69,7 @@ func VerifyProof(rh util.Uint256, key []byte, proofs [][]byte) ([]byte, bool) {
tr := NewTrie(NewHashNode(rh), ModeAll, storage.NewMemCachedStore(storage.NewMemoryStore())) tr := NewTrie(NewHashNode(rh), ModeAll, storage.NewMemCachedStore(storage.NewMemoryStore()))
for i := range proofs { for i := range proofs {
h := hash.DoubleSha256(proofs[i]) h := hash.DoubleSha256(proofs[i])
// no errors in Put to memory store tr.Store.Put(makeStorageKey(h), proofs[i])
_ = tr.Store.Put(makeStorageKey(h), proofs[i])
} }
_, leaf, _, err := tr.getWithPath(tr.root, path, true) _, leaf, _, err := tr.getWithPath(tr.root, path, true)
if err != nil { if err != nil {

8
pkg/core/mpt/trie.go
View file

@ -420,7 +420,7 @@ func (t *Trie) Flush(index uint32) {
delete(t.refcount, h) delete(t.refcount, h)
} }
} else if node.refcount > 0 { } else if node.refcount > 0 {
_ = t.Store.Put(key, node.bytes) t.Store.Put(key, node.bytes)
} }
node.refcount = 0 node.refcount = 0
} else { } else {
@ -467,15 +467,15 @@ func (t *Trie) updateRefCount(h util.Uint256, key []byte, index uint32) int32 {
panic(fmt.Sprintf("negative reference count: %s new %d, upd %d", h.StringBE(), cnt, t.refcount[h])) panic(fmt.Sprintf("negative reference count: %s new %d, upd %d", h.StringBE(), cnt, t.refcount[h]))
case cnt == 0: case cnt == 0:
if !t.mode.GC() { if !t.mode.GC() {
_ = t.Store.Delete(key) t.Store.Delete(key)
} else { } else {
data[len(data)-5] = 0 data[len(data)-5] = 0
binary.LittleEndian.PutUint32(data[len(data)-4:], index) binary.LittleEndian.PutUint32(data[len(data)-4:], index)
_ = t.Store.Put(key, data) t.Store.Put(key, data)
} }
default: default:
binary.LittleEndian.PutUint32(data[len(data)-4:], uint32(cnt)) binary.LittleEndian.PutUint32(data[len(data)-4:], uint32(cnt))
_ = t.Store.Put(key, data) t.Store.Put(key, data)
} }
return cnt return cnt
} }

2
pkg/core/mpt/trie_test.go
View file

@ -251,7 +251,7 @@ func (tr *Trie) putToStore(n Node) {
} }
tr.updateRefCount(n.Hash(), makeStorageKey(n.Hash()), 0) tr.updateRefCount(n.Hash(), makeStorageKey(n.Hash()), 0)
} else { } else {
_ = tr.Store.Put(makeStorageKey(n.Hash()), n.Bytes()) tr.Store.Put(makeStorageKey(n.Hash()), n.Bytes())
} }
} }

6
pkg/core/native/designate.go
View file

@ -180,7 +180,7 @@ func (s *Designate) hashFromNodes(r noderoles.Role, nodes keys.PublicKeys) util.
return hash.Hash160(script) return hash.Hash160(script)
} }
func (s *Designate) updateCachedRoleData(v *atomic.Value, d dao.DAO, r noderoles.Role) error { func (s *Designate) updateCachedRoleData(v *atomic.Value, d *dao.Simple, r noderoles.Role) error {
nodeKeys, height, err := s.GetDesignatedByRole(d, r, math.MaxUint32) nodeKeys, height, err := s.GetDesignatedByRole(d, r, math.MaxUint32)
if err != nil { if err != nil {
return err return err
@ -226,7 +226,7 @@ func (s *Designate) getCachedRoleData(r noderoles.Role) *roleData {
} }
// GetLastDesignatedHash returns last designated hash of a given role. // GetLastDesignatedHash returns last designated hash of a given role.
func (s *Designate) GetLastDesignatedHash(d dao.DAO, r noderoles.Role) (util.Uint160, error) { func (s *Designate) GetLastDesignatedHash(d *dao.Simple, r noderoles.Role) (util.Uint160, error) {
if !s.isValidRole(r) { if !s.isValidRole(r) {
return util.Uint160{}, ErrInvalidRole return util.Uint160{}, ErrInvalidRole
} }
@ -244,7 +244,7 @@ func (s *Designate) GetLastDesignatedHash(d dao.DAO, r noderoles.Role) (util.Uin
} }
// GetDesignatedByRole returns nodes for role r. // GetDesignatedByRole returns nodes for role r.
func (s *Designate) GetDesignatedByRole(d dao.DAO, r noderoles.Role, index uint32) (keys.PublicKeys, uint32, error) { func (s *Designate) GetDesignatedByRole(d *dao.Simple, r noderoles.Role, index uint32) (keys.PublicKeys, uint32, error) {
if !s.isValidRole(r) { if !s.isValidRole(r) {
return nil, 0, ErrInvalidRole return nil, 0, ErrInvalidRole
} }

2
pkg/core/native/ledger.go
View file

@ -179,7 +179,7 @@ func getBlockHashFromItem(bc interop.Ledger, item stackitem.Item) util.Uint256 {
// getTransactionAndHeight returns transaction and its height if it's present // getTransactionAndHeight returns transaction and its height if it's present
// on the chain. It panics if anything goes wrong. // on the chain. It panics if anything goes wrong.
func getTransactionAndHeight(d dao.DAO, item stackitem.Item) (*transaction.Transaction, uint32, error) { func getTransactionAndHeight(d *dao.Simple, item stackitem.Item) (*transaction.Transaction, uint32, error) {
hashbytes, err := item.TryBytes() hashbytes, err := item.TryBytes()
if err != nil { if err != nil {
panic(err) panic(err)

51
pkg/core/native/management.go
View file

@ -145,7 +145,7 @@ func (m *Management) getContract(ic *interop.Context, args []stackitem.Item) sta
} }
// GetContract returns contract with given hash from given DAO. // GetContract returns contract with given hash from given DAO.
func (m *Management) GetContract(d dao.DAO, hash util.Uint160) (*state.Contract, error) { func (m *Management) GetContract(d *dao.Simple, hash util.Uint160) (*state.Contract, error) {
m.mtx.RLock() m.mtx.RLock()
cs, ok := m.contracts[hash] cs, ok := m.contracts[hash]
m.mtx.RUnlock() m.mtx.RUnlock()
@ -157,7 +157,7 @@ func (m *Management) GetContract(d dao.DAO, hash util.Uint160) (*state.Contract,
return m.getContractFromDAO(d, hash) return m.getContractFromDAO(d, hash)
} }
func (m *Management) getContractFromDAO(d dao.DAO, hash util.Uint160) (*state.Contract, error) { func (m *Management) getContractFromDAO(d *dao.Simple, hash util.Uint160) (*state.Contract, error) {
contract := new(state.Contract) contract := new(state.Contract)
key := MakeContractKey(hash) key := MakeContractKey(hash)
err := getConvertibleFromDAO(m.ID, d, key, contract) err := getConvertibleFromDAO(m.ID, d, key, contract)
@ -269,7 +269,7 @@ func (m *Management) markUpdated(h util.Uint160) {
// Deploy creates contract's hash/ID and saves new contract into the given DAO. // Deploy creates contract's hash/ID and saves new contract into the given DAO.
// It doesn't run _deploy method and doesn't emit notification. // It doesn't run _deploy method and doesn't emit notification.
func (m *Management) Deploy(d dao.DAO, sender util.Uint160, neff *nef.File, manif *manifest.Manifest) (*state.Contract, error) { func (m *Management) Deploy(d *dao.Simple, sender util.Uint160, neff *nef.File, manif *manifest.Manifest) (*state.Contract, error) {
h := state.CreateContractHash(sender, neff.Checksum, manif.Name) h := state.CreateContractHash(sender, neff.Checksum, manif.Name)
key := MakeContractKey(h) key := MakeContractKey(h)
si := d.GetStorageItem(m.ID, key) si := d.GetStorageItem(m.ID, key)
@ -329,7 +329,7 @@ func (m *Management) updateWithData(ic *interop.Context, args []stackitem.Item)
// Update updates contract's script and/or manifest in the given DAO. // Update updates contract's script and/or manifest in the given DAO.
// It doesn't run _deploy method and doesn't emit notification. // It doesn't run _deploy method and doesn't emit notification.
func (m *Management) Update(d dao.DAO, hash util.Uint160, neff *nef.File, manif *manifest.Manifest) (*state.Contract, error) { func (m *Management) Update(d *dao.Simple, hash util.Uint160, neff *nef.File, manif *manifest.Manifest) (*state.Contract, error) {
var contract state.Contract var contract state.Contract
oldcontract, err := m.GetContract(d, hash) oldcontract, err := m.GetContract(d, hash)
@ -380,29 +380,20 @@ func (m *Management) destroy(ic *interop.Context, sis []stackitem.Item) stackite
} }
// Destroy drops given contract from DAO along with its storage. It doesn't emit notification. // Destroy drops given contract from DAO along with its storage. It doesn't emit notification.
func (m *Management) Destroy(d dao.DAO, hash util.Uint160) error { func (m *Management) Destroy(d *dao.Simple, hash util.Uint160) error {
contract, err := m.GetContract(d, hash) contract, err := m.GetContract(d, hash)
if err != nil { if err != nil {
return err return err
} }
key := MakeContractKey(hash) key := MakeContractKey(hash)
err = d.DeleteStorageItem(m.ID, key) d.DeleteStorageItem(m.ID, key)
if err != nil { d.DeleteContractID(contract.ID)
return err
}
err = d.DeleteContractID(contract.ID)
if err != nil {
return err
}
siArr, err := d.GetStorageItems(contract.ID) siArr, err := d.GetStorageItems(contract.ID)
if err != nil { if err != nil {
return err return err
} }
for _, kv := range siArr { for _, kv := range siArr {
err := d.DeleteStorageItem(contract.ID, []byte(kv.Key)) d.DeleteStorageItem(contract.ID, []byte(kv.Key))
if err != nil {
return err
}
} }
m.markUpdated(hash) m.markUpdated(hash)
return nil return nil
@ -413,7 +404,7 @@ func (m *Management) getMinimumDeploymentFee(ic *interop.Context, args []stackit
} }
// GetMinimumDeploymentFee returns the minimum required fee for contract deploy. // GetMinimumDeploymentFee returns the minimum required fee for contract deploy.
func (m *Management) GetMinimumDeploymentFee(dao dao.DAO) int64 { func (m *Management) GetMinimumDeploymentFee(dao *dao.Simple) int64 {
return getIntWithKey(m.ID, dao, keyMinimumDeploymentFee) return getIntWithKey(m.ID, dao, keyMinimumDeploymentFee)
} }
@ -425,10 +416,7 @@ func (m *Management) setMinimumDeploymentFee(ic *interop.Context, args []stackit
if !m.NEO.checkCommittee(ic) { if !m.NEO.checkCommittee(ic) {
panic("invalid committee signature") panic("invalid committee signature")
} }
err := ic.DAO.PutStorageItem(m.ID, keyMinimumDeploymentFee, bigint.ToBytes(value)) ic.DAO.PutStorageItem(m.ID, keyMinimumDeploymentFee, bigint.ToBytes(value))
if err != nil {
panic(err)
}
return stackitem.Null{} return stackitem.Null{}
} }
@ -498,7 +486,7 @@ func (m *Management) OnPersist(ic *interop.Context) error {
// InitializeCache initializes contract cache with the proper values from storage. // InitializeCache initializes contract cache with the proper values from storage.
// Cache initialisation should be done apart from Initialize because Initialize is // Cache initialisation should be done apart from Initialize because Initialize is
// called only when deploying native contracts. // called only when deploying native contracts.
func (m *Management) InitializeCache(d dao.DAO) error { func (m *Management) InitializeCache(d *dao.Simple) error {
m.mtx.Lock() m.mtx.Lock()
defer m.mtx.Unlock() defer m.mtx.Unlock()
@ -564,14 +552,13 @@ func (m *Management) GetNEP17Contracts() []util.Uint160 {
// Initialize implements Contract interface. // Initialize implements Contract interface.
func (m *Management) Initialize(ic *interop.Context) error { func (m *Management) Initialize(ic *interop.Context) error {
if err := setIntWithKey(m.ID, ic.DAO, keyMinimumDeploymentFee, defaultMinimumDeploymentFee); err != nil { setIntWithKey(m.ID, ic.DAO, keyMinimumDeploymentFee, defaultMinimumDeploymentFee)
return err setIntWithKey(m.ID, ic.DAO, keyNextAvailableID, 1)
} return nil
return setIntWithKey(m.ID, ic.DAO, keyNextAvailableID, 1)
} }
// PutContractState saves given contract state into given DAO. // PutContractState saves given contract state into given DAO.
func (m *Management) PutContractState(d dao.DAO, cs *state.Contract) error { func (m *Management) PutContractState(d *dao.Simple, cs *state.Contract) error {
key := MakeContractKey(cs.Hash) key := MakeContractKey(cs.Hash)
if err := putConvertibleToDAO(m.ID, d, key, cs); err != nil { if err := putConvertibleToDAO(m.ID, d, key, cs); err != nil {
return err return err
@ -580,10 +567,11 @@ func (m *Management) PutContractState(d dao.DAO, cs *state.Contract) error {
if cs.UpdateCounter != 0 { // Update. if cs.UpdateCounter != 0 { // Update.
return nil return nil
} }
return d.PutContractID(cs.ID, cs.Hash) d.PutContractID(cs.ID, cs.Hash)
return nil
} }
func (m *Management) getNextContractID(d dao.DAO) (int32, error) { func (m *Management) getNextContractID(d *dao.Simple) (int32, error) {
si := d.GetStorageItem(m.ID, keyNextAvailableID) si := d.GetStorageItem(m.ID, keyNextAvailableID)
if si == nil { if si == nil {
return 0, errors.New("nextAvailableID is not initialized") return 0, errors.New("nextAvailableID is not initialized")
@ -592,7 +580,8 @@ func (m *Management) getNextContractID(d dao.DAO) (int32, error) {
ret := int32(id.Int64()) ret := int32(id.Int64())
id.Add(id, intOne) id.Add(id, intOne)
si = bigint.ToPreallocatedBytes(id, si) si = bigint.ToPreallocatedBytes(id, si)
return ret, d.PutStorageItem(m.ID, keyNextAvailableID, si) d.PutStorageItem(m.ID, keyNextAvailableID, si)
return ret, nil
} }
func (m *Management) emitNotification(ic *interop.Context, name string, hash util.Uint160) { func (m *Management) emitNotification(ic *interop.Context, name string, hash util.Uint160) {

2
pkg/core/native/management_test.go
View file

@ -79,7 +79,7 @@ func TestManagement_Initialize(t *testing.T) {
t.Run("invalid contract state", func(t *testing.T) { t.Run("invalid contract state", func(t *testing.T) {
d := dao.NewSimple(storage.NewMemoryStore(), false, false) d := dao.NewSimple(storage.NewMemoryStore(), false, false)
mgmt := newManagement() mgmt := newManagement()
require.NoError(t, d.PutStorageItem(mgmt.ID, []byte{prefixContract}, state.StorageItem{0xFF})) d.PutStorageItem(mgmt.ID, []byte{prefixContract}, state.StorageItem{0xFF})
require.Error(t, mgmt.InitializeCache(d)) require.Error(t, mgmt.InitializeCache(d))
}) })
} }

2
pkg/core/native/native_gas.go
View file

@ -131,7 +131,7 @@ func (g *GAS) PostPersist(ic *interop.Context) error {
} }
// BalanceOf returns native GAS token balance for the acc. // BalanceOf returns native GAS token balance for the acc.
func (g *GAS) BalanceOf(d dao.DAO, acc util.Uint160) *big.Int { func (g *GAS) BalanceOf(d *dao.Simple, acc util.Uint160) *big.Int {
return g.balanceOfInternal(d, acc) return g.balanceOfInternal(d, acc)
} }

104
pkg/core/native/native_neo.go
View file

@ -215,10 +215,7 @@ func (n *NEO) Initialize(ic *interop.Context) error {
return err return err
} }
err = ic.DAO.PutStorageItem(n.ID, prefixCommittee, cvs.Bytes()) ic.DAO.PutStorageItem(n.ID, prefixCommittee, cvs.Bytes())
if err != nil {
return err
}
h, err := getStandbyValidatorsHash(ic) h, err := getStandbyValidatorsHash(ic)
if err != nil { if err != nil {
@ -228,22 +225,14 @@ func (n *NEO) Initialize(ic *interop.Context) error {
var index uint32 var index uint32
value := big.NewInt(5 * GASFactor) value := big.NewInt(5 * GASFactor)
err = n.putGASRecord(ic.DAO, index, value) n.putGASRecord(ic.DAO, index, value)
if err != nil {
return err
}
gr := &gasRecord{{Index: index, GASPerBlock: *value}} gr := &gasRecord{{Index: index, GASPerBlock: *value}}
n.gasPerBlock.Store(*gr) n.gasPerBlock.Store(*gr)
n.gasPerBlockChanged.Store(false) n.gasPerBlockChanged.Store(false)
err = ic.DAO.PutStorageItem(n.ID, []byte{prefixVotersCount}, state.StorageItem{}) ic.DAO.PutStorageItem(n.ID, []byte{prefixVotersCount}, state.StorageItem{})
if err != nil {
return err
}
err = setIntWithKey(n.ID, ic.DAO, []byte{prefixRegisterPrice}, DefaultRegisterPrice) setIntWithKey(n.ID, ic.DAO, []byte{prefixRegisterPrice}, DefaultRegisterPrice)
if err != nil {
return err
}
n.registerPrice.Store(int64(DefaultRegisterPrice)) n.registerPrice.Store(int64(DefaultRegisterPrice))
n.registerPriceChanged.Store(false) n.registerPriceChanged.Store(false)
return nil return nil
@ -252,7 +241,7 @@ func (n *NEO) Initialize(ic *interop.Context) error {
// InitializeCache initializes all NEO cache with the proper values from storage. // InitializeCache initializes all NEO cache with the proper values from storage.
// Cache initialisation should be done apart from Initialize because Initialize is // Cache initialisation should be done apart from Initialize because Initialize is
// called only when deploying native contracts. // called only when deploying native contracts.
func (n *NEO) InitializeCache(bc interop.Ledger, d dao.DAO) error { func (n *NEO) InitializeCache(bc interop.Ledger, d *dao.Simple) error {
err := n.initConfigCache(bc) err := n.initConfigCache(bc)
if err != nil { if err != nil {
return nil return nil
@ -305,7 +294,8 @@ func (n *NEO) updateCommittee(ic *interop.Context) error {
if !votesChanged { if !votesChanged {
// We need to put in storage anyway, as it affects dumps // We need to put in storage anyway, as it affects dumps
committee := n.committee.Load().(keysWithVotes) committee := n.committee.Load().(keysWithVotes)
return ic.DAO.PutStorageItem(n.ID, prefixCommittee, committee.Bytes()) ic.DAO.PutStorageItem(n.ID, prefixCommittee, committee.Bytes())
return nil
} }
_, cvs, err := n.computeCommitteeMembers(ic.Chain, ic.DAO) _, cvs, err := n.computeCommitteeMembers(ic.Chain, ic.DAO)
@ -316,7 +306,8 @@ func (n *NEO) updateCommittee(ic *interop.Context) error {
return err return err
} }
n.votesChanged.Store(false) n.votesChanged.Store(false)
return ic.DAO.PutStorageItem(n.ID, prefixCommittee, cvs.Bytes()) ic.DAO.PutStorageItem(n.ID, prefixCommittee, cvs.Bytes())
return nil
} }
// OnPersist implements Contract interface. // OnPersist implements Contract interface.
@ -379,9 +370,7 @@ func (n *NEO) PostPersist(ic *interop.Context) error {
binary.BigEndian.PutUint32(key[34:], ic.Block.Index+1) binary.BigEndian.PutUint32(key[34:], ic.Block.Index+1)
n.gasPerVoteCache[cs[i].Key] = *tmp n.gasPerVoteCache[cs[i].Key] = *tmp
if err := ic.DAO.PutStorageItem(n.ID, key, bigint.ToBytes(tmp)); err != nil { ic.DAO.PutStorageItem(n.ID, key, bigint.ToBytes(tmp))
return err
}
} }
} }
} }
@ -402,7 +391,7 @@ func (n *NEO) PostPersist(ic *interop.Context) error {
return nil return nil
} }
func (n *NEO) getGASPerVote(d dao.DAO, key []byte, indexes []uint32) []big.Int { func (n *NEO) getGASPerVote(d *dao.Simple, key []byte, indexes []uint32) []big.Int {
sort.Slice(indexes, func(i, j int) bool { sort.Slice(indexes, func(i, j int) bool {
return indexes[i] < indexes[j] return indexes[i] < indexes[j]
}) })
@ -485,10 +474,7 @@ func (n *NEO) distributeGas(ic *interop.Context, h util.Uint160, acc *state.NEOB
// Must store acc before GAS distribution to fix acc's BalanceHeight value in the storage for // Must store acc before GAS distribution to fix acc's BalanceHeight value in the storage for
// further acc's queries from `onNEP17Payment` if so, see https://github.com/nspcc-dev/neo-go/pull/2181. // further acc's queries from `onNEP17Payment` if so, see https://github.com/nspcc-dev/neo-go/pull/2181.
key := makeAccountKey(h) key := makeAccountKey(h)
err = ic.DAO.PutStorageItem(n.ID, key, acc.Bytes()) ic.DAO.PutStorageItem(n.ID, key, acc.Bytes())
if err != nil {
return fmt.Errorf("failed to store acc before gas distribution: %w", err)
}
n.GAS.mint(ic, h, gen, true) n.GAS.mint(ic, h, gen, true)
return nil return nil
@ -509,7 +495,7 @@ func (n *NEO) getGASPerBlock(ic *interop.Context, _ []stackitem.Item) stackitem.
return stackitem.NewBigInteger(gas) return stackitem.NewBigInteger(gas)
} }
func (n *NEO) getSortedGASRecordFromDAO(d dao.DAO) (gasRecord, error) { func (n *NEO) getSortedGASRecordFromDAO(d *dao.Simple) (gasRecord, error) {
grArr, err := d.GetStorageItemsWithPrefix(n.ID, []byte{prefixGASPerBlock}) grArr, err := d.GetStorageItemsWithPrefix(n.ID, []byte{prefixGASPerBlock})
if err != nil { if err != nil {
return gasRecord{}, fmt.Errorf("failed to get gas records from storage: %w", err) return gasRecord{}, fmt.Errorf("failed to get gas records from storage: %w", err)
@ -528,7 +514,7 @@ func (n *NEO) getSortedGASRecordFromDAO(d dao.DAO) (gasRecord, error) {
} }
// GetGASPerBlock returns gas generated for block with provided index. // GetGASPerBlock returns gas generated for block with provided index.
func (n *NEO) GetGASPerBlock(d dao.DAO, index uint32) *big.Int { func (n *NEO) GetGASPerBlock(d *dao.Simple, index uint32) *big.Int {
var ( var (
gr gasRecord gr gasRecord
err error err error
@ -581,14 +567,15 @@ func (n *NEO) SetGASPerBlock(ic *interop.Context, index uint32, gas *big.Int) er
return errors.New("invalid committee signature") return errors.New("invalid committee signature")
} }
n.gasPerBlockChanged.Store(true) n.gasPerBlockChanged.Store(true)
return n.putGASRecord(ic.DAO, index, gas) n.putGASRecord(ic.DAO, index, gas)
return nil
} }
func (n *NEO) getRegisterPrice(ic *interop.Context, _ []stackitem.Item) stackitem.Item { func (n *NEO) getRegisterPrice(ic *interop.Context, _ []stackitem.Item) stackitem.Item {
return stackitem.NewBigInteger(big.NewInt(n.getRegisterPriceInternal(ic.DAO))) return stackitem.NewBigInteger(big.NewInt(n.getRegisterPriceInternal(ic.DAO)))
} }
func (n *NEO) getRegisterPriceInternal(d dao.DAO) int64 { func (n *NEO) getRegisterPriceInternal(d *dao.Simple) int64 {
if !n.registerPriceChanged.Load().(bool) { if !n.registerPriceChanged.Load().(bool) {
return n.registerPrice.Load().(int64) return n.registerPrice.Load().(int64)
} }
@ -604,21 +591,16 @@ func (n *NEO) setRegisterPrice(ic *interop.Context, args []stackitem.Item) stack
panic("invalid committee signature") panic("invalid committee signature")
} }
err := setIntWithKey(n.ID, ic.DAO, []byte{prefixRegisterPrice}, price.Int64()) setIntWithKey(n.ID, ic.DAO, []byte{prefixRegisterPrice}, price.Int64())
if err != nil {
panic(err)
}
n.registerPriceChanged.Store(true) n.registerPriceChanged.Store(true)
return stackitem.Null{} return stackitem.Null{}
} }
func (n *NEO) dropCandidateIfZero(d dao.DAO, pub *keys.PublicKey, c *candidate) (bool, error) { func (n *NEO) dropCandidateIfZero(d *dao.Simple, pub *keys.PublicKey, c *candidate) (bool, error) {
if c.Registered || c.Votes.Sign() != 0 { if c.Registered || c.Votes.Sign() != 0 {
return false, nil return false, nil
} }
if err := d.DeleteStorageItem(n.ID, makeValidatorKey(pub)); err != nil { d.DeleteStorageItem(n.ID, makeValidatorKey(pub))
return true, err
}
var toRemove []string var toRemove []string
voterKey := makeVoterKey(pub.Bytes()) voterKey := makeVoterKey(pub.Bytes())
@ -627,9 +609,7 @@ func (n *NEO) dropCandidateIfZero(d dao.DAO, pub *keys.PublicKey, c *candidate)
return true return true
}) })
for i := range toRemove { for i := range toRemove {
if err := d.DeleteStorageItem(n.ID, []byte(toRemove[i])); err != nil { d.DeleteStorageItem(n.ID, []byte(toRemove[i]))
return true, err
}
} }
delete(n.gasPerVoteCache, string(voterKey)) delete(n.gasPerVoteCache, string(voterKey))
@ -650,7 +630,7 @@ func makeVoterKey(pub []byte, prealloc ...[]byte) []byte {
// CalculateBonus calculates amount of gas generated for holding value NEO from start to end block // CalculateBonus calculates amount of gas generated for holding value NEO from start to end block
// and having voted for active committee member. // and having voted for active committee member.
func (n *NEO) CalculateBonus(d dao.DAO, acc util.Uint160, end uint32) (*big.Int, error) { func (n *NEO) CalculateBonus(d *dao.Simple, acc util.Uint160, end uint32) (*big.Int, error) {
key := makeAccountKey(acc) key := makeAccountKey(acc)
si := d.GetStorageItem(n.ID, key) si := d.GetStorageItem(n.ID, key)
if si == nil { if si == nil {
@ -663,7 +643,7 @@ func (n *NEO) CalculateBonus(d dao.DAO, acc util.Uint160, end uint32) (*big.Int,
return n.calculateBonus(d, st.VoteTo, &st.Balance, st.BalanceHeight, end) return n.calculateBonus(d, st.VoteTo, &st.Balance, st.BalanceHeight, end)
} }
func (n *NEO) calculateBonus(d dao.DAO, vote *keys.PublicKey, value *big.Int, start, end uint32) (*big.Int, error) { func (n *NEO) calculateBonus(d *dao.Simple, vote *keys.PublicKey, value *big.Int, start, end uint32) (*big.Int, error) {
r, err := n.CalculateNEOHolderReward(d, value, start, end) r, err := n.CalculateNEOHolderReward(d, value, start, end)
if err != nil || vote == nil { if err != nil || vote == nil {
return r, err return r, err
@ -679,7 +659,7 @@ func (n *NEO) calculateBonus(d dao.DAO, vote *keys.PublicKey, value *big.Int, st
} }
// CalculateNEOHolderReward return GAS reward for holding `value` of NEO from start to end block. // CalculateNEOHolderReward return GAS reward for holding `value` of NEO from start to end block.
func (n *NEO) CalculateNEOHolderReward(d dao.DAO, value *big.Int, start, end uint32) (*big.Int, error) { func (n *NEO) CalculateNEOHolderReward(d *dao.Simple, value *big.Int, start, end uint32) (*big.Int, error) {
if value.Sign() == 0 || start >= end { if value.Sign() == 0 || start >= end {
return big.NewInt(0), nil return big.NewInt(0), nil
} else if value.Sign() < 0 { } else if value.Sign() < 0 {
@ -804,10 +784,7 @@ func (n *NEO) VoteInternal(ic *interop.Context, h util.Uint160, pub *keys.Public
return err return err
} }
// we should put it in storage anyway as it affects dumps // we should put it in storage anyway as it affects dumps
err = ic.DAO.PutStorageItem(n.ID, key, si) ic.DAO.PutStorageItem(n.ID, key, si)
if err != nil {
return err
}
if pub != nil { if pub != nil {
valKey := makeValidatorKey(pub) valKey := makeValidatorKey(pub)
valSi := ic.DAO.GetStorageItem(n.ID, valKey) valSi := ic.DAO.GetStorageItem(n.ID, valKey)
@ -816,10 +793,7 @@ func (n *NEO) VoteInternal(ic *interop.Context, h util.Uint160, pub *keys.Public
} }
cd := new(candidate).FromBytes(valSi) cd := new(candidate).FromBytes(valSi)
// we should put it in storage anyway as it affects dumps // we should put it in storage anyway as it affects dumps
err = ic.DAO.PutStorageItem(n.ID, valKey, valSi) ic.DAO.PutStorageItem(n.ID, valKey, valSi)
if err != nil {
return err
}
if !cd.Registered { if !cd.Registered {
return errors.New("validator must be registered") return errors.New("validator must be registered")
} }
@ -844,12 +818,13 @@ func (n *NEO) VoteInternal(ic *interop.Context, h util.Uint160, pub *keys.Public
if err := n.ModifyAccountVotes(acc, ic.DAO, &acc.Balance, true); err != nil { if err := n.ModifyAccountVotes(acc, ic.DAO, &acc.Balance, true); err != nil {
return err return err
} }
return ic.DAO.PutStorageItem(n.ID, key, acc.Bytes()) ic.DAO.PutStorageItem(n.ID, key, acc.Bytes())
return nil
} }
// ModifyAccountVotes modifies votes of the specified account by value (can be negative). // ModifyAccountVotes modifies votes of the specified account by value (can be negative).
// typ specifies if this modify is occurring during transfer or vote (with old or new validator). // typ specifies if this modify is occurring during transfer or vote (with old or new validator).
func (n *NEO) ModifyAccountVotes(acc *state.NEOBalance, d dao.DAO, value *big.Int, isNewVote bool) error { func (n *NEO) ModifyAccountVotes(acc *state.NEOBalance, d *dao.Simple, value *big.Int, isNewVote bool) error {
n.votesChanged.Store(true) n.votesChanged.Store(true)
if acc.VoteTo != nil { if acc.VoteTo != nil {
key := makeValidatorKey(acc.VoteTo) key := makeValidatorKey(acc.VoteTo)
@ -871,7 +846,7 @@ func (n *NEO) ModifyAccountVotes(acc *state.NEOBalance, d dao.DAO, value *big.In
return nil return nil
} }
func (n *NEO) getCandidates(d dao.DAO, sortByKey bool) ([]keyWithVotes, error) { func (n *NEO) getCandidates(d *dao.Simple, sortByKey bool) ([]keyWithVotes, error) {
siArr, err := d.GetStorageItemsWithPrefix(n.ID, []byte{prefixCandidate}) siArr, err := d.GetStorageItemsWithPrefix(n.ID, []byte{prefixCandidate})
if err != nil { if err != nil {
return nil, err return nil, err
@ -911,7 +886,7 @@ func (n *NEO) getCandidates(d dao.DAO, sortByKey bool) ([]keyWithVotes, error) {
// GetCandidates returns current registered validators list with keys // GetCandidates returns current registered validators list with keys
// and votes. // and votes.
func (n *NEO) GetCandidates(d dao.DAO) ([]state.Validator, error) { func (n *NEO) GetCandidates(d *dao.Simple) ([]state.Validator, error) {
kvs, err := n.getCandidates(d, true) kvs, err := n.getCandidates(d, true)
if err != nil { if err != nil {
return nil, err return nil, err
@ -957,7 +932,7 @@ func (n *NEO) getAccountState(ic *interop.Context, args []stackitem.Item) stacki
} }
// ComputeNextBlockValidators returns an actual list of current validators. // ComputeNextBlockValidators returns an actual list of current validators.
func (n *NEO) ComputeNextBlockValidators(bc interop.Ledger, d dao.DAO) (keys.PublicKeys, error) { func (n *NEO) ComputeNextBlockValidators(bc interop.Ledger, d *dao.Simple) (keys.PublicKeys, error) {
numOfCNs := n.cfg.GetNumOfCNs(bc.BlockHeight() + 1) numOfCNs := n.cfg.GetNumOfCNs(bc.BlockHeight() + 1)
if vals := n.validators.Load().(keys.PublicKeys); vals != nil && numOfCNs == len(vals) { if vals := n.validators.Load().(keys.PublicKeys); vals != nil && numOfCNs == len(vals) {
return vals.Copy(), nil return vals.Copy(), nil
@ -978,7 +953,7 @@ func (n *NEO) getCommittee(ic *interop.Context, _ []stackitem.Item) stackitem.It
return pubsToArray(pubs) return pubsToArray(pubs)
} }
func (n *NEO) modifyVoterTurnout(d dao.DAO, amount *big.Int) error { func (n *NEO) modifyVoterTurnout(d *dao.Simple, amount *big.Int) error {
key := []byte{prefixVotersCount} key := []byte{prefixVotersCount}
si := d.GetStorageItem(n.ID, key) si := d.GetStorageItem(n.ID, key)
if si == nil { if si == nil {
@ -987,7 +962,8 @@ func (n *NEO) modifyVoterTurnout(d dao.DAO, amount *big.Int) error {
votersCount := bigint.FromBytes(si) votersCount := bigint.FromBytes(si)
votersCount.Add(votersCount, amount) votersCount.Add(votersCount, amount)
si = bigint.ToPreallocatedBytes(votersCount, si) si = bigint.ToPreallocatedBytes(votersCount, si)
return d.PutStorageItem(n.ID, key, si) d.PutStorageItem(n.ID, key, si)
return nil
} }
// GetCommitteeMembers returns public keys of nodes in committee using cached value. // GetCommitteeMembers returns public keys of nodes in committee using cached value.
@ -1015,7 +991,7 @@ func toKeysWithVotes(pubs keys.PublicKeys) keysWithVotes {
} }
// computeCommitteeMembers returns public keys of nodes in committee. // computeCommitteeMembers returns public keys of nodes in committee.
func (n *NEO) computeCommitteeMembers(bc interop.Ledger, d dao.DAO) (keys.PublicKeys, keysWithVotes, error) { func (n *NEO) computeCommitteeMembers(bc interop.Ledger, d *dao.Simple) (keys.PublicKeys, keysWithVotes, error) {
key := []byte{prefixVotersCount} key := []byte{prefixVotersCount}
si := d.GetStorageItem(n.ID, key) si := d.GetStorageItem(n.ID, key)
if si == nil { if si == nil {
@ -1071,7 +1047,7 @@ func (n *NEO) GetNextBlockValidatorsInternal() keys.PublicKeys {
} }
// BalanceOf returns native NEO token balance for the acc. // BalanceOf returns native NEO token balance for the acc.
func (n *NEO) BalanceOf(d dao.DAO, acc util.Uint160) (*big.Int, uint32) { func (n *NEO) BalanceOf(d *dao.Simple, acc util.Uint160) (*big.Int, uint32) {
key := makeAccountKey(acc) key := makeAccountKey(acc)
si := d.GetStorageItem(n.ID, key) si := d.GetStorageItem(n.ID, key)
if si == nil { if si == nil {
@ -1105,9 +1081,9 @@ func toPublicKey(s stackitem.Item) *keys.PublicKey {
} }
// putGASRecord is a helper which creates key and puts GASPerBlock value into the storage. // putGASRecord is a helper which creates key and puts GASPerBlock value into the storage.
func (n *NEO) putGASRecord(dao dao.DAO, index uint32, value *big.Int) error { func (n *NEO) putGASRecord(dao *dao.Simple, index uint32, value *big.Int) {
key := make([]byte, 5) key := make([]byte, 5)
key[0] = prefixGASPerBlock key[0] = prefixGASPerBlock
binary.BigEndian.PutUint32(key[1:], index) binary.BigEndian.PutUint32(key[1:], index)
return dao.PutStorageItem(n.ID, key, bigint.ToBytes(value)) dao.PutStorageItem(n.ID, key, bigint.ToBytes(value))
} }

29
pkg/core/native/native_nep17.go
View file

@ -99,7 +99,7 @@ func (c *nep17TokenNative) TotalSupply(ic *interop.Context, _ []stackitem.Item)
return stackitem.NewBigInteger(supply) return stackitem.NewBigInteger(supply)
} }
func (c *nep17TokenNative) getTotalSupply(d dao.DAO) (state.StorageItem, *big.Int) { func (c *nep17TokenNative) getTotalSupply(d *dao.Simple) (state.StorageItem, *big.Int) {
si := d.GetStorageItem(c.ID, totalSupplyKey) si := d.GetStorageItem(c.ID, totalSupplyKey)
if si == nil { if si == nil {
si = []byte{} si = []byte{}
@ -107,9 +107,9 @@ func (c *nep17TokenNative) getTotalSupply(d dao.DAO) (state.StorageItem, *big.In
return si, bigint.FromBytes(si) return si, bigint.FromBytes(si)
} }
func (c *nep17TokenNative) saveTotalSupply(d dao.DAO, si state.StorageItem, supply *big.Int) error { func (c *nep17TokenNative) saveTotalSupply(d *dao.Simple, si state.StorageItem, supply *big.Int) {
si = state.StorageItem(bigint.ToPreallocatedBytes(supply, si)) si = state.StorageItem(bigint.ToPreallocatedBytes(supply, si))
return d.PutStorageItem(c.ID, totalSupplyKey, si) d.PutStorageItem(c.ID, totalSupplyKey, si)
} }
func (c *nep17TokenNative) Transfer(ic *interop.Context, args []stackitem.Item) stackitem.Item { func (c *nep17TokenNative) Transfer(ic *interop.Context, args []stackitem.Item) stackitem.Item {
@ -184,16 +184,16 @@ func (c *nep17TokenNative) updateAccBalance(ic *interop.Context, acc util.Uint16
err := c.incBalance(ic, acc, &si, amount, requiredBalance) err := c.incBalance(ic, acc, &si, amount, requiredBalance)
if err != nil { if err != nil {
if si != nil && amount.Sign() <= 0 { if si != nil && amount.Sign() <= 0 {
_ = ic.DAO.PutStorageItem(c.ID, key, si) ic.DAO.PutStorageItem(c.ID, key, si)
} }
return err return err
} }
if si == nil { if si == nil {
err = ic.DAO.DeleteStorageItem(c.ID, key) ic.DAO.DeleteStorageItem(c.ID, key)
} else { } else {
err = ic.DAO.PutStorageItem(c.ID, key, si) ic.DAO.PutStorageItem(c.ID, key, si)
} }
return err return nil
} }
// TransferInternal transfers NEO between accounts. // TransferInternal transfers NEO between accounts.
@ -237,7 +237,7 @@ func (c *nep17TokenNative) balanceOf(ic *interop.Context, args []stackitem.Item)
return stackitem.NewBigInteger(c.balanceOfInternal(ic.DAO, h)) return stackitem.NewBigInteger(c.balanceOfInternal(ic.DAO, h))
} }
func (c *nep17TokenNative) balanceOfInternal(d dao.DAO, h util.Uint160) *big.Int { func (c *nep17TokenNative) balanceOfInternal(d *dao.Simple, h util.Uint160) *big.Int {
key := makeAccountKey(h) key := makeAccountKey(h)
si := d.GetStorageItem(c.ID, key) si := d.GetStorageItem(c.ID, key)
if si == nil { if si == nil {
@ -281,22 +281,15 @@ func (c *nep17TokenNative) addTokens(ic *interop.Context, h util.Uint160, amount
if err := c.incBalance(ic, h, &si, amount, nil); err != nil { if err := c.incBalance(ic, h, &si, amount, nil); err != nil {
panic(err) panic(err)
} }
var err error
if si == nil { if si == nil {
err = ic.DAO.DeleteStorageItem(c.ID, key) ic.DAO.DeleteStorageItem(c.ID, key)
} else { } else {
err = ic.DAO.PutStorageItem(c.ID, key, si) ic.DAO.PutStorageItem(c.ID, key, si)
}
if err != nil {
panic(err)
} }
buf, supply := c.getTotalSupply(ic.DAO) buf, supply := c.getTotalSupply(ic.DAO)
supply.Add(supply, amount) supply.Add(supply, amount)
err = c.saveTotalSupply(ic.DAO, buf, supply) c.saveTotalSupply(ic.DAO, buf, supply)
if err != nil {
panic(err)
}
} }
func newDescriptor(name string, ret smartcontract.ParamType, ps ...manifest.Parameter) *manifest.Method { func newDescriptor(name string, ret smartcontract.ParamType, ps ...manifest.Parameter) *manifest.Method {

36
pkg/core/native/notary.go
View file

@ -109,11 +109,7 @@ func (n *Notary) Metadata() *interop.ContractMD {
// Initialize initializes Notary native contract and implements Contract interface. // Initialize initializes Notary native contract and implements Contract interface.
func (n *Notary) Initialize(ic *interop.Context) error { func (n *Notary) Initialize(ic *interop.Context) error {
err := setIntWithKey(n.ID, ic.DAO, maxNotValidBeforeDeltaKey, defaultMaxNotValidBeforeDelta) setIntWithKey(n.ID, ic.DAO, maxNotValidBeforeDeltaKey, defaultMaxNotValidBeforeDelta)
if err != nil {
return err
}
n.isValid = true n.isValid = true
n.maxNotValidBeforeDelta = defaultMaxNotValidBeforeDelta n.maxNotValidBeforeDelta = defaultMaxNotValidBeforeDelta
return nil return nil
@ -141,10 +137,7 @@ func (n *Notary) OnPersist(ic *interop.Context) error {
balance := n.GetDepositFor(ic.DAO, payer.Account) balance := n.GetDepositFor(ic.DAO, payer.Account)
balance.Amount.Sub(balance.Amount, big.NewInt(tx.SystemFee+tx.NetworkFee)) balance.Amount.Sub(balance.Amount, big.NewInt(tx.SystemFee+tx.NetworkFee))
if balance.Amount.Sign() == 0 { if balance.Amount.Sign() == 0 {
err := n.removeDepositFor(ic.DAO, payer.Account) n.removeDepositFor(ic.DAO, payer.Account)
if err != nil {
return fmt.Errorf("failed to remove an empty deposit for %s from storage: %w", payer.Account.StringBE(), err)
}
} else { } else {
err := n.putDepositFor(ic.DAO, balance, payer.Account) err := n.putDepositFor(ic.DAO, balance, payer.Account)
if err != nil { if err != nil {
@ -289,9 +282,7 @@ func (n *Notary) withdraw(ic *interop.Context, args []stackitem.Item) stackitem.
if !ic.VM.Estack().Pop().Bool() { if !ic.VM.Estack().Pop().Bool() {
panic("failed to transfer GAS from Notary account: `transfer` returned false") panic("failed to transfer GAS from Notary account: `transfer` returned false")
} }
if err := n.removeDepositFor(ic.DAO, from); err != nil { n.removeDepositFor(ic.DAO, from)
panic(fmt.Errorf("failed to remove withdrawn deposit for %s from the storage: %w", from.StringBE(), err))
}
return stackitem.NewBool(true) return stackitem.NewBool(true)
} }
@ -302,7 +293,7 @@ func (n *Notary) balanceOf(ic *interop.Context, args []stackitem.Item) stackitem
} }
// BalanceOf is an internal representation of `balanceOf` Notary method. // BalanceOf is an internal representation of `balanceOf` Notary method.
func (n *Notary) BalanceOf(dao dao.DAO, acc util.Uint160) *big.Int { func (n *Notary) BalanceOf(dao *dao.Simple, acc util.Uint160) *big.Int {
deposit := n.GetDepositFor(dao, acc) deposit := n.GetDepositFor(dao, acc)
if deposit == nil { if deposit == nil {
return big.NewInt(0) return big.NewInt(0)
@ -317,7 +308,7 @@ func (n *Notary) expirationOf(ic *interop.Context, args []stackitem.Item) stacki
} }
// ExpirationOf is an internal representation of `expirationOf` Notary method. // ExpirationOf is an internal representation of `expirationOf` Notary method.
func (n *Notary) ExpirationOf(dao dao.DAO, acc util.Uint160) uint32 { func (n *Notary) ExpirationOf(dao *dao.Simple, acc util.Uint160) uint32 {
deposit := n.GetDepositFor(dao, acc) deposit := n.GetDepositFor(dao, acc)
if deposit == nil { if deposit == nil {
return 0 return 0
@ -369,7 +360,7 @@ func (n *Notary) verify(ic *interop.Context, args []stackitem.Item) stackitem.It
} }
// GetNotaryNodes returns public keys of notary nodes. // GetNotaryNodes returns public keys of notary nodes.
func (n *Notary) GetNotaryNodes(d dao.DAO) (keys.PublicKeys, error) { func (n *Notary) GetNotaryNodes(d *dao.Simple) (keys.PublicKeys, error) {
nodes, _, err := n.Desig.GetDesignatedByRole(d, noderoles.P2PNotary, math.MaxUint32) nodes, _, err := n.Desig.GetDesignatedByRole(d, noderoles.P2PNotary, math.MaxUint32)
return nodes, err return nodes, err
} }
@ -380,7 +371,7 @@ func (n *Notary) getMaxNotValidBeforeDelta(ic *interop.Context, _ []stackitem.It
} }
// GetMaxNotValidBeforeDelta is an internal representation of Notary getMaxNotValidBeforeDelta method. // GetMaxNotValidBeforeDelta is an internal representation of Notary getMaxNotValidBeforeDelta method.
func (n *Notary) GetMaxNotValidBeforeDelta(dao dao.DAO) uint32 { func (n *Notary) GetMaxNotValidBeforeDelta(dao *dao.Simple) uint32 {
n.lock.RLock() n.lock.RLock()
defer n.lock.RUnlock() defer n.lock.RUnlock()
if n.isValid { if n.isValid {
@ -402,17 +393,14 @@ func (n *Notary) setMaxNotValidBeforeDelta(ic *interop.Context, args []stackitem
} }
n.lock.Lock() n.lock.Lock()
defer n.lock.Unlock() defer n.lock.Unlock()
err := setIntWithKey(n.ID, ic.DAO, maxNotValidBeforeDeltaKey, int64(value)) setIntWithKey(n.ID, ic.DAO, maxNotValidBeforeDeltaKey, int64(value))
if err != nil {
panic(fmt.Errorf("failed to put value into the storage: %w", err))
}
n.isValid = false n.isValid = false
return stackitem.Null{} return stackitem.Null{}
} }
// GetDepositFor returns state.Deposit for the account specified. It returns nil in case if // GetDepositFor returns state.Deposit for the account specified. It returns nil in case if
// deposit is not found in storage and panics in case of any other error. // deposit is not found in storage and panics in case of any other error.
func (n *Notary) GetDepositFor(dao dao.DAO, acc util.Uint160) *state.Deposit { func (n *Notary) GetDepositFor(dao *dao.Simple, acc util.Uint160) *state.Deposit {
key := append([]byte{prefixDeposit}, acc.BytesBE()...) key := append([]byte{prefixDeposit}, acc.BytesBE()...)
deposit := new(state.Deposit) deposit := new(state.Deposit)
err := getConvertibleFromDAO(n.ID, dao, key, deposit) err := getConvertibleFromDAO(n.ID, dao, key, deposit)
@ -426,15 +414,15 @@ func (n *Notary) GetDepositFor(dao dao.DAO, acc util.Uint160) *state.Deposit {
} }
// putDepositFor puts deposit on the balance of the specified account in the storage. // putDepositFor puts deposit on the balance of the specified account in the storage.
func (n *Notary) putDepositFor(dao dao.DAO, deposit *state.Deposit, acc util.Uint160) error { func (n *Notary) putDepositFor(dao *dao.Simple, deposit *state.Deposit, acc util.Uint160) error {
key := append([]byte{prefixDeposit}, acc.BytesBE()...) key := append([]byte{prefixDeposit}, acc.BytesBE()...)
return putConvertibleToDAO(n.ID, dao, key, deposit) return putConvertibleToDAO(n.ID, dao, key, deposit)
} }
// removeDepositFor removes deposit from the storage. // removeDepositFor removes deposit from the storage.
func (n *Notary) removeDepositFor(dao dao.DAO, acc util.Uint160) error { func (n *Notary) removeDepositFor(dao *dao.Simple, acc util.Uint160) {
key := append([]byte{prefixDeposit}, acc.BytesBE()...) key := append([]byte{prefixDeposit}, acc.BytesBE()...)
return dao.DeleteStorageItem(n.ID, key) dao.DeleteStorageItem(n.ID, key)
} }
// calculateNotaryReward calculates the reward for a single notary node based on FEE's count and Notary nodes count. // calculateNotaryReward calculates the reward for a single notary node based on FEE's count and Notary nodes count.

42
pkg/core/native/oracle.go
View file

@ -166,9 +166,7 @@ func (o *Oracle) PostPersist(ic *interop.Context) error {
if err := o.getConvertibleFromDAO(ic.DAO, reqKey, req); err != nil { if err := o.getConvertibleFromDAO(ic.DAO, reqKey, req); err != nil {
continue continue
} }
if err := ic.DAO.DeleteStorageItem(o.ID, reqKey); err != nil { ic.DAO.DeleteStorageItem(o.ID, reqKey)
return err
}
if orc != nil { if orc != nil {
removedIDs = append(removedIDs, resp.ID) removedIDs = append(removedIDs, resp.ID)
} }
@ -184,7 +182,7 @@ func (o *Oracle) PostPersist(ic *interop.Context) error {
var err error var err error
if len(*idList) == 0 { if len(*idList) == 0 {
err = ic.DAO.DeleteStorageItem(o.ID, idKey) ic.DAO.DeleteStorageItem(o.ID, idKey)
} else { } else {
err = putConvertibleToDAO(o.ID, ic.DAO, idKey, idList) err = putConvertibleToDAO(o.ID, ic.DAO, idKey, idList)
} }
@ -222,12 +220,8 @@ func (o *Oracle) Metadata() *interop.ContractMD {
// Initialize initializes Oracle contract. // Initialize initializes Oracle contract.
func (o *Oracle) Initialize(ic *interop.Context) error { func (o *Oracle) Initialize(ic *interop.Context) error {
if err := setIntWithKey(o.ID, ic.DAO, prefixRequestID, 0); err != nil { setIntWithKey(o.ID, ic.DAO, prefixRequestID, 0)
return err setIntWithKey(o.ID, ic.DAO, prefixRequestPrice, DefaultOracleRequestPrice)
}
if err := setIntWithKey(o.ID, ic.DAO, prefixRequestPrice, DefaultOracleRequestPrice); err != nil {
return err
}
o.requestPrice.Store(int64(DefaultOracleRequestPrice)) o.requestPrice.Store(int64(DefaultOracleRequestPrice))
o.requestPriceChanged.Store(false) o.requestPriceChanged.Store(false)
return nil return nil
@ -348,9 +342,7 @@ func (o *Oracle) RequestInternal(ic *interop.Context, url string, filter *string
id := itemID.Uint64() id := itemID.Uint64()
itemID.Add(itemID, intOne) itemID.Add(itemID, intOne)
si = bigint.ToPreallocatedBytes(itemID, si) si = bigint.ToPreallocatedBytes(itemID, si)
if err := ic.DAO.PutStorageItem(o.ID, prefixRequestID, si); err != nil { ic.DAO.PutStorageItem(o.ID, prefixRequestID, si)
return err
}
// Should be executed from contract. // Should be executed from contract.
_, err := ic.GetContract(ic.VM.GetCallingScriptHash()) _, err := ic.GetContract(ic.VM.GetCallingScriptHash())
@ -395,7 +387,7 @@ func (o *Oracle) RequestInternal(ic *interop.Context, url string, filter *string
} }
// PutRequestInternal puts oracle request with the specified id to d. // PutRequestInternal puts oracle request with the specified id to d.
func (o *Oracle) PutRequestInternal(id uint64, req *state.OracleRequest, d dao.DAO) error { func (o *Oracle) PutRequestInternal(id uint64, req *state.OracleRequest, d *dao.Simple) error {
reqKey := makeRequestKey(id) reqKey := makeRequestKey(id)
if err := putConvertibleToDAO(o.ID, d, reqKey, req); err != nil { if err := putConvertibleToDAO(o.ID, d, reqKey, req); err != nil {
return err return err
@ -416,25 +408,25 @@ func (o *Oracle) PutRequestInternal(id uint64, req *state.OracleRequest, d dao.D
} }
// GetScriptHash returns script hash or oracle nodes. // GetScriptHash returns script hash or oracle nodes.
func (o *Oracle) GetScriptHash(d dao.DAO) (util.Uint160, error) { func (o *Oracle) GetScriptHash(d *dao.Simple) (util.Uint160, error) {
return o.Desig.GetLastDesignatedHash(d, noderoles.Oracle) return o.Desig.GetLastDesignatedHash(d, noderoles.Oracle)
} }
// GetOracleNodes returns public keys of oracle nodes. // GetOracleNodes returns public keys of oracle nodes.
func (o *Oracle) GetOracleNodes(d dao.DAO) (keys.PublicKeys, error) { func (o *Oracle) GetOracleNodes(d *dao.Simple) (keys.PublicKeys, error) {
nodes, _, err := o.Desig.GetDesignatedByRole(d, noderoles.Oracle, math.MaxUint32) nodes, _, err := o.Desig.GetDesignatedByRole(d, noderoles.Oracle, math.MaxUint32)
return nodes, err return nodes, err
} }
// GetRequestInternal returns request by ID and key under which it is stored. // GetRequestInternal returns request by ID and key under which it is stored.
func (o *Oracle) GetRequestInternal(d dao.DAO, id uint64) (*state.OracleRequest, error) { func (o *Oracle) GetRequestInternal(d *dao.Simple, id uint64) (*state.OracleRequest, error) {
key := makeRequestKey(id) key := makeRequestKey(id)
req := new(state.OracleRequest) req := new(state.OracleRequest)
return req, o.getConvertibleFromDAO(d, key, req) return req, o.getConvertibleFromDAO(d, key, req)
} }
// GetIDListInternal returns request by ID and key under which it is stored. // GetIDListInternal returns request by ID and key under which it is stored.
func (o *Oracle) GetIDListInternal(d dao.DAO, url string) (*IDList, error) { func (o *Oracle) GetIDListInternal(d *dao.Simple, url string) (*IDList, error) {
key := makeIDListKey(url) key := makeIDListKey(url)
idList := new(IDList) idList := new(IDList)
return idList, o.getConvertibleFromDAO(d, key, idList) return idList, o.getConvertibleFromDAO(d, key, idList)
@ -448,7 +440,7 @@ func (o *Oracle) getPrice(ic *interop.Context, _ []stackitem.Item) stackitem.Ite
return stackitem.NewBigInteger(big.NewInt(o.getPriceInternal(ic.DAO))) return stackitem.NewBigInteger(big.NewInt(o.getPriceInternal(ic.DAO)))
} }
func (o *Oracle) getPriceInternal(d dao.DAO) int64 { func (o *Oracle) getPriceInternal(d *dao.Simple) int64 {
if !o.requestPriceChanged.Load().(bool) { if !o.requestPriceChanged.Load().(bool) {
return o.requestPrice.Load().(int64) return o.requestPrice.Load().(int64)
} }
@ -463,14 +455,12 @@ func (o *Oracle) setPrice(ic *interop.Context, args []stackitem.Item) stackitem.
if !o.NEO.checkCommittee(ic) { if !o.NEO.checkCommittee(ic) {
panic("invalid committee signature") panic("invalid committee signature")
} }
if err := setIntWithKey(o.ID, ic.DAO, prefixRequestPrice, price.Int64()); err != nil { setIntWithKey(o.ID, ic.DAO, prefixRequestPrice, price.Int64())
panic(err)
}
o.requestPriceChanged.Store(true) o.requestPriceChanged.Store(true)
return stackitem.Null{} return stackitem.Null{}
} }
func (o *Oracle) getOriginalTxID(d dao.DAO, tx *transaction.Transaction) util.Uint256 { func (o *Oracle) getOriginalTxID(d *dao.Simple, tx *transaction.Transaction) util.Uint256 {
for i := range tx.Attributes { for i := range tx.Attributes {
if tx.Attributes[i].Type == transaction.OracleResponseT { if tx.Attributes[i].Type == transaction.OracleResponseT {
id := tx.Attributes[i].Value.(*transaction.OracleResponse).ID id := tx.Attributes[i].Value.(*transaction.OracleResponse).ID
@ -482,7 +472,7 @@ func (o *Oracle) getOriginalTxID(d dao.DAO, tx *transaction.Transaction) util.Ui
} }
// getRequests returns all requests which have not been finished yet. // getRequests returns all requests which have not been finished yet.
func (o *Oracle) getRequests(d dao.DAO) (map[uint64]*state.OracleRequest, error) { func (o *Oracle) getRequests(d *dao.Simple) (map[uint64]*state.OracleRequest, error) {
arr, err := d.GetStorageItemsWithPrefix(o.ID, prefixRequest) arr, err := d.GetStorageItemsWithPrefix(o.ID, prefixRequest)
if err != nil { if err != nil {
return nil, err return nil, err
@ -514,12 +504,12 @@ func makeIDListKey(url string) []byte {
return append(prefixIDList, hash.Hash160([]byte(url)).BytesBE()...) return append(prefixIDList, hash.Hash160([]byte(url)).BytesBE()...)
} }
func (o *Oracle) getConvertibleFromDAO(d dao.DAO, key []byte, item stackitem.Convertible) error { func (o *Oracle) getConvertibleFromDAO(d *dao.Simple, key []byte, item stackitem.Convertible) error {
return getConvertibleFromDAO(o.ID, d, key, item) return getConvertibleFromDAO(o.ID, d, key, item)
} }
// updateCache updates cached Oracle values if they've been changed. // updateCache updates cached Oracle values if they've been changed.
func (o *Oracle) updateCache(d dao.DAO) error { func (o *Oracle) updateCache(d *dao.Simple) error {
orc, _ := o.Module.Load().(services.Oracle) orc, _ := o.Module.Load().(services.Oracle)
if orc == nil { if orc == nil {
return nil return nil

49
pkg/core/native/policy.go
View file

@ -123,15 +123,9 @@ func (p *Policy) Metadata() *interop.ContractMD {
// Initialize initializes Policy native contract and implements Contract interface. // Initialize initializes Policy native contract and implements Contract interface.
func (p *Policy) Initialize(ic *interop.Context) error { func (p *Policy) Initialize(ic *interop.Context) error {
if err := setIntWithKey(p.ID, ic.DAO, feePerByteKey, defaultFeePerByte); err != nil { setIntWithKey(p.ID, ic.DAO, feePerByteKey, defaultFeePerByte)
return err setIntWithKey(p.ID, ic.DAO, execFeeFactorKey, defaultExecFeeFactor)
} setIntWithKey(p.ID, ic.DAO, storagePriceKey, DefaultStoragePrice)
if err := setIntWithKey(p.ID, ic.DAO, execFeeFactorKey, defaultExecFeeFactor); err != nil {
return err
}
if err := setIntWithKey(p.ID, ic.DAO, storagePriceKey, DefaultStoragePrice); err != nil {
return err
}
p.isValid = true p.isValid = true
p.execFeeFactor = defaultExecFeeFactor p.execFeeFactor = defaultExecFeeFactor
@ -188,7 +182,7 @@ func (p *Policy) getFeePerByte(ic *interop.Context, _ []stackitem.Item) stackite
} }
// GetFeePerByteInternal returns required transaction's fee per byte. // GetFeePerByteInternal returns required transaction's fee per byte.
func (p *Policy) GetFeePerByteInternal(dao dao.DAO) int64 { func (p *Policy) GetFeePerByteInternal(dao *dao.Simple) int64 {
p.lock.RLock() p.lock.RLock()
defer p.lock.RUnlock() defer p.lock.RUnlock()
if p.isValid { if p.isValid {
@ -198,7 +192,7 @@ func (p *Policy) GetFeePerByteInternal(dao dao.DAO) int64 {
} }
// GetMaxVerificationGas returns maximum gas allowed to be burned during verificaion. // GetMaxVerificationGas returns maximum gas allowed to be burned during verificaion.
func (p *Policy) GetMaxVerificationGas(_ dao.DAO) int64 { func (p *Policy) GetMaxVerificationGas(_ *dao.Simple) int64 {
if p.isValid { if p.isValid {
return p.maxVerificationGas return p.maxVerificationGas
} }
@ -210,7 +204,7 @@ func (p *Policy) getExecFeeFactor(ic *interop.Context, _ []stackitem.Item) stack
} }
// GetExecFeeFactorInternal returns current execution fee factor. // GetExecFeeFactorInternal returns current execution fee factor.
func (p *Policy) GetExecFeeFactorInternal(d dao.DAO) int64 { func (p *Policy) GetExecFeeFactorInternal(d *dao.Simple) int64 {
p.lock.RLock() p.lock.RLock()
defer p.lock.RUnlock() defer p.lock.RUnlock()
if p.isValid { if p.isValid {
@ -229,10 +223,7 @@ func (p *Policy) setExecFeeFactor(ic *interop.Context, args []stackitem.Item) st
} }
p.lock.Lock() p.lock.Lock()
defer p.lock.Unlock() defer p.lock.Unlock()
err := setIntWithKey(p.ID, ic.DAO, execFeeFactorKey, int64(value)) setIntWithKey(p.ID, ic.DAO, execFeeFactorKey, int64(value))
if err != nil {
panic(err)
}
p.isValid = false p.isValid = false
return stackitem.Null{} return stackitem.Null{}
} }
@ -244,7 +235,7 @@ func (p *Policy) isBlocked(ic *interop.Context, args []stackitem.Item) stackitem
} }
// IsBlockedInternal checks whether provided account is blocked. // IsBlockedInternal checks whether provided account is blocked.
func (p *Policy) IsBlockedInternal(dao dao.DAO, hash util.Uint160) bool { func (p *Policy) IsBlockedInternal(dao *dao.Simple, hash util.Uint160) bool {
p.lock.RLock() p.lock.RLock()
defer p.lock.RUnlock() defer p.lock.RUnlock()
if p.isValid { if p.isValid {
@ -266,7 +257,7 @@ func (p *Policy) getStoragePrice(ic *interop.Context, _ []stackitem.Item) stacki
} }
// GetStoragePriceInternal returns current execution fee factor. // GetStoragePriceInternal returns current execution fee factor.
func (p *Policy) GetStoragePriceInternal(d dao.DAO) int64 { func (p *Policy) GetStoragePriceInternal(d *dao.Simple) int64 {
p.lock.RLock() p.lock.RLock()
defer p.lock.RUnlock() defer p.lock.RUnlock()
if p.isValid { if p.isValid {
@ -285,10 +276,7 @@ func (p *Policy) setStoragePrice(ic *interop.Context, args []stackitem.Item) sta
} }
p.lock.Lock() p.lock.Lock()
defer p.lock.Unlock() defer p.lock.Unlock()
err := setIntWithKey(p.ID, ic.DAO, storagePriceKey, int64(value)) setIntWithKey(p.ID, ic.DAO, storagePriceKey, int64(value))
if err != nil {
panic(err)
}
p.isValid = false p.isValid = false
return stackitem.Null{} return stackitem.Null{}
} }
@ -304,10 +292,7 @@ func (p *Policy) setFeePerByte(ic *interop.Context, args []stackitem.Item) stack
} }
p.lock.Lock() p.lock.Lock()
defer p.lock.Unlock() defer p.lock.Unlock()
err := setIntWithKey(p.ID, ic.DAO, feePerByteKey, value) setIntWithKey(p.ID, ic.DAO, feePerByteKey, value)
if err != nil {
panic(err)
}
p.isValid = false p.isValid = false
return stackitem.Null{} return stackitem.Null{}
} }
@ -330,10 +315,7 @@ func (p *Policy) blockAccount(ic *interop.Context, args []stackitem.Item) stacki
key := append([]byte{blockedAccountPrefix}, hash.BytesBE()...) key := append([]byte{blockedAccountPrefix}, hash.BytesBE()...)
p.lock.Lock() p.lock.Lock()
defer p.lock.Unlock() defer p.lock.Unlock()
err := ic.DAO.PutStorageItem(p.ID, key, state.StorageItem{}) ic.DAO.PutStorageItem(p.ID, key, state.StorageItem{})
if err != nil {
panic(err)
}
p.isValid = false p.isValid = false
return stackitem.NewBool(true) return stackitem.NewBool(true)
} }
@ -351,10 +333,7 @@ func (p *Policy) unblockAccount(ic *interop.Context, args []stackitem.Item) stac
key := append([]byte{blockedAccountPrefix}, hash.BytesBE()...) key := append([]byte{blockedAccountPrefix}, hash.BytesBE()...)
p.lock.Lock() p.lock.Lock()
defer p.lock.Unlock() defer p.lock.Unlock()
err := ic.DAO.DeleteStorageItem(p.ID, key) ic.DAO.DeleteStorageItem(p.ID, key)
if err != nil {
panic(err)
}
p.isValid = false p.isValid = false
return stackitem.NewBool(true) return stackitem.NewBool(true)
} }
@ -362,7 +341,7 @@ func (p *Policy) unblockAccount(ic *interop.Context, args []stackitem.Item) stac
// CheckPolicy checks whether transaction conforms to current policy restrictions // CheckPolicy checks whether transaction conforms to current policy restrictions
// like not being signed by blocked account or not exceeding block-level system // like not being signed by blocked account or not exceeding block-level system
// fee limit. // fee limit.
func (p *Policy) CheckPolicy(d dao.DAO, tx *transaction.Transaction) error { func (p *Policy) CheckPolicy(d *dao.Simple, tx *transaction.Transaction) error {
for _, signer := range tx.Signers { for _, signer := range tx.Signers {
if p.IsBlockedInternal(d, signer.Account) { if p.IsBlockedInternal(d, signer.Account) {
return fmt.Errorf("account %s is blocked", signer.Account.StringLE()) return fmt.Errorf("account %s is blocked", signer.Account.StringLE())

13
pkg/core/native/util.go
View file

@ -15,7 +15,7 @@ import (
var intOne = big.NewInt(1) var intOne = big.NewInt(1)
var intTwo = big.NewInt(2) var intTwo = big.NewInt(2)
func getConvertibleFromDAO(id int32, d dao.DAO, key []byte, conv stackitem.Convertible) error { func getConvertibleFromDAO(id int32, d *dao.Simple, key []byte, conv stackitem.Convertible) error {
si := d.GetStorageItem(id, key) si := d.GetStorageItem(id, key)
if si == nil { if si == nil {
return storage.ErrKeyNotFound return storage.ErrKeyNotFound
@ -23,19 +23,20 @@ func getConvertibleFromDAO(id int32, d dao.DAO, key []byte, conv stackitem.Conve
return stackitem.DeserializeConvertible(si, conv) return stackitem.DeserializeConvertible(si, conv)
} }
func putConvertibleToDAO(id int32, d dao.DAO, key []byte, conv stackitem.Convertible) error { func putConvertibleToDAO(id int32, d *dao.Simple, key []byte, conv stackitem.Convertible) error {
data, err := stackitem.SerializeConvertible(conv) data, err := stackitem.SerializeConvertible(conv)
if err != nil { if err != nil {
return err return err
} }
return d.PutStorageItem(id, key, data) d.PutStorageItem(id, key, data)
return nil
} }
func setIntWithKey(id int32, dao dao.DAO, key []byte, value int64) error { func setIntWithKey(id int32, dao *dao.Simple, key []byte, value int64) {
return dao.PutStorageItem(id, key, bigint.ToBytes(big.NewInt(value))) dao.PutStorageItem(id, key, bigint.ToBytes(big.NewInt(value)))
} }
func getIntWithKey(id int32, dao dao.DAO, key []byte) int64 { func getIntWithKey(id int32, dao *dao.Simple, key []byte) int64 {
si := dao.GetStorageItem(id, key) si := dao.GetStorageItem(id, key)
if si == nil { if si == nil {
panic(fmt.Errorf("item with id = %d and key = %s is not initialized", id, hex.EncodeToString(key))) panic(fmt.Errorf("item with id = %d and key = %s is not initialized", id, hex.EncodeToString(key)))

22
pkg/core/stateroot/module.go
View file

@ -146,13 +146,13 @@ func (s *Module) CleanStorage() error {
if s.localHeight.Load() != 0 { if s.localHeight.Load() != 0 {
return fmt.Errorf("can't clean MPT data for non-genesis block: expected local stateroot height 0, got %d", s.localHeight.Load()) return fmt.Errorf("can't clean MPT data for non-genesis block: expected local stateroot height 0, got %d", s.localHeight.Load())
} }
b := s.Store.Batch() b := storage.NewMemCachedStore(s.Store)
s.Store.Seek(storage.SeekRange{Prefix: []byte{byte(storage.DataMPT)}}, func(k, _ []byte) bool { s.Store.Seek(storage.SeekRange{Prefix: []byte{byte(storage.DataMPT)}}, func(k, _ []byte) bool {
// #1468, but don't need to copy here, because it is done by Store. // #1468, but don't need to copy here, because it is done by Store.
b.Delete(k) b.Delete(k)
return true return true
}) })
err := s.Store.PutBatch(b) _, err := b.Persist()
if err != nil { if err != nil {
return fmt.Errorf("failed to remove outdated MPT-reated items: %w", err) return fmt.Errorf("failed to remove outdated MPT-reated items: %w", err)
} }
@ -160,22 +160,17 @@ func (s *Module) CleanStorage() error {
} }
// JumpToState performs jump to the state specified by given stateroot index. // JumpToState performs jump to the state specified by given stateroot index.
func (s *Module) JumpToState(sr *state.MPTRoot) error { func (s *Module) JumpToState(sr *state.MPTRoot) {
if err := s.addLocalStateRoot(s.Store, sr); err != nil { s.addLocalStateRoot(s.Store, sr)
return fmt.Errorf("failed to store local state root: %w", err)
}
data := make([]byte, 4) data := make([]byte, 4)
binary.LittleEndian.PutUint32(data, sr.Index) binary.LittleEndian.PutUint32(data, sr.Index)
if err := s.Store.Put([]byte{byte(storage.DataMPTAux), prefixValidated}, data); err != nil { s.Store.Put([]byte{byte(storage.DataMPTAux), prefixValidated}, data)
return fmt.Errorf("failed to store validated height: %w", err)
}
s.validatedHeight.Store(sr.Index) s.validatedHeight.Store(sr.Index)
s.currentLocal.Store(sr.Root) s.currentLocal.Store(sr.Root)
s.localHeight.Store(sr.Index) s.localHeight.Store(sr.Index)
s.mpt = mpt.NewTrie(mpt.NewHashNode(sr.Root), s.mode, s.Store) s.mpt = mpt.NewTrie(mpt.NewHashNode(sr.Root), s.mode, s.Store)
return nil
} }
// GC performs garbage collection. // GC performs garbage collection.
@ -225,11 +220,8 @@ func (s *Module) AddMPTBatch(index uint32, b mpt.Batch, cache *storage.MemCached
Index: index, Index: index,
Root: mpt.StateRoot(), Root: mpt.StateRoot(),
} }
err := s.addLocalStateRoot(cache, sr) s.addLocalStateRoot(cache, sr)
if err != nil { return &mpt, sr, nil
return nil, nil, err
}
return &mpt, sr, err
} }
// UpdateCurrentLocal updates local caches using provided state root. // UpdateCurrentLocal updates local caches using provided state root.

20
pkg/core/stateroot/store.go
View file

@ -21,21 +21,19 @@ const (
prefixValidated = 0x03 prefixValidated = 0x03
) )
func (s *Module) addLocalStateRoot(store *storage.MemCachedStore, sr *state.MPTRoot) error { func (s *Module) addLocalStateRoot(store *storage.MemCachedStore, sr *state.MPTRoot) {
key := makeStateRootKey(sr.Index) key := makeStateRootKey(sr.Index)
if err := putStateRoot(store, key, sr); err != nil { putStateRoot(store, key, sr)
return err
}
data := make([]byte, 4) data := make([]byte, 4)
binary.LittleEndian.PutUint32(data, sr.Index) binary.LittleEndian.PutUint32(data, sr.Index)
return store.Put([]byte{byte(storage.DataMPTAux), prefixLocal}, data) store.Put([]byte{byte(storage.DataMPTAux), prefixLocal}, data)
} }
func putStateRoot(store *storage.MemCachedStore, key []byte, sr *state.MPTRoot) error { func putStateRoot(store *storage.MemCachedStore, key []byte, sr *state.MPTRoot) {
w := io.NewBufBinWriter() w := io.NewBufBinWriter()
sr.EncodeBinary(w.BinWriter) sr.EncodeBinary(w.BinWriter)
return store.Put(key, w.Bytes()) store.Put(key, w.Bytes())
} }
func (s *Module) getStateRoot(key []byte) (*state.MPTRoot, error) { func (s *Module) getStateRoot(key []byte) (*state.MPTRoot, error) {
@ -73,15 +71,11 @@ func (s *Module) AddStateRoot(sr *state.MPTRoot) error {
if len(local.Witness) != 0 { if len(local.Witness) != 0 {
return nil return nil
} }
if err := putStateRoot(s.Store, key, sr); err != nil { putStateRoot(s.Store, key, sr)
return err
}
data := make([]byte, 4) data := make([]byte, 4)
binary.LittleEndian.PutUint32(data, sr.Index) binary.LittleEndian.PutUint32(data, sr.Index)
if err := s.Store.Put([]byte{byte(storage.DataMPTAux), prefixValidated}, data); err != nil { s.Store.Put([]byte{byte(storage.DataMPTAux), prefixValidated}, data)
return err
}
s.validatedHeight.Store(sr.Index) s.validatedHeight.Store(sr.Index)
if !s.srInHead { if !s.srInHead {
updateStateHeightMetric(sr.Index) updateStateHeightMetric(sr.Index)

21
pkg/core/statesync/module.go
View file

@ -167,10 +167,7 @@ func (s *Module) Init(currChainHeight uint32) error {
} }
s.syncPoint = p s.syncPoint = p
err = s.dao.PutStateSyncPoint(p) s.dao.PutStateSyncPoint(p)
if err != nil {
return fmt.Errorf("failed to store state synchronisation point %d: %w", p, err)
}
s.syncStage = initialized s.syncStage = initialized
s.log.Info("try to sync state for the latest state synchronisation point", s.log.Info("try to sync state for the latest state synchronisation point",
zap.Uint32("point", p), zap.Uint32("point", p),
@ -332,26 +329,20 @@ func (s *Module) AddBlock(block *block.Block) error {
return errors.New("invalid block: MerkleRoot mismatch") return errors.New("invalid block: MerkleRoot mismatch")
} }
} }
cache := s.dao.GetWrapped() cache := s.dao.GetPrivate()
writeBuf := io.NewBufBinWriter() if err := cache.StoreAsBlock(block, nil, nil); err != nil {
if err := cache.StoreAsBlock(block, nil, nil, writeBuf); err != nil {
return err return err
} }
writeBuf.Reset()
err := cache.PutStateSyncCurrentBlockHeight(block.Index) cache.PutStateSyncCurrentBlockHeight(block.Index)
if err != nil {
return fmt.Errorf("failed to store current block height: %w", err)
}
for _, tx := range block.Transactions { for _, tx := range block.Transactions {
if err := cache.StoreAsTransaction(tx, block.Index, nil, writeBuf); err != nil { if err := cache.StoreAsTransaction(tx, block.Index, nil); err != nil {
return err return err
} }
writeBuf.Reset()
} }
_, err = cache.Persist() _, err := cache.Persist()
if err != nil { if err != nil {
return fmt.Errorf("failed to persist results: %w", err) return fmt.Errorf("failed to persist results: %w", err)
} }

35
pkg/core/storage/boltdb_store.go
View file

@ -47,15 +47,6 @@ func NewBoltDBStore(cfg BoltDBOptions) (*BoltDBStore, error) {
return &BoltDBStore{db: db}, nil return &BoltDBStore{db: db}, nil
} }
// Put implements the Store interface.
func (s *BoltDBStore) Put(key, value []byte) error {
return s.db.Update(func(tx *bbolt.Tx) error {
b := tx.Bucket(Bucket)
err := b.Put(key, value)
return err
})
}
// Get implements the Store interface. // Get implements the Store interface.
func (s *BoltDBStore) Get(key []byte) (val []byte, err error) { func (s *BoltDBStore) Get(key []byte) (val []byte, err error) {
err = s.db.View(func(tx *bbolt.Tx) error { err = s.db.View(func(tx *bbolt.Tx) error {
@ -73,27 +64,14 @@ func (s *BoltDBStore) Get(key []byte) (val []byte, err error) {
return return
} }
// Delete implements the Store interface.
func (s *BoltDBStore) Delete(key []byte) error {
return s.db.Update(func(tx *bbolt.Tx) error {
b := tx.Bucket(Bucket)
return b.Delete(key)
})
}
// PutBatch implements the Store interface.
func (s *BoltDBStore) PutBatch(batch Batch) error {
memBatch := batch.(*MemoryBatch)
return s.PutChangeSet(memBatch.mem)
}
// PutChangeSet implements the Store interface. // PutChangeSet implements the Store interface.
func (s *BoltDBStore) PutChangeSet(puts map[string][]byte) error { func (s *BoltDBStore) PutChangeSet(puts map[string][]byte, stores map[string][]byte) error {
var err error var err error
return s.db.Update(func(tx *bbolt.Tx) error { return s.db.Update(func(tx *bbolt.Tx) error {
b := tx.Bucket(Bucket) b := tx.Bucket(Bucket)
for k, v := range puts { for _, m := range []map[string][]byte{puts, stores} {
for k, v := range m {
if v != nil { if v != nil {
err = b.Put([]byte(k), v) err = b.Put([]byte(k), v)
} else { } else {
@ -103,6 +81,7 @@ func (s *BoltDBStore) PutChangeSet(puts map[string][]byte) error {
return err return err
} }
} }
}
return nil return nil
}) })
} }
@ -166,12 +145,6 @@ func boltSeek(txopener func(func(*bbolt.Tx) error) error, rng SeekRange, f func(
}) })
} }
// Batch implements the Batch interface and returns a boltdb
// compatible Batch.
func (s *BoltDBStore) Batch() Batch {
return newMemoryBatch()
}
// Close releases all db resources. // Close releases all db resources.
func (s *BoltDBStore) Close() error { func (s *BoltDBStore) Close() error {
return s.db.Close() return s.db.Close()

32
pkg/core/storage/leveldb_store.go
View file

@ -36,11 +36,6 @@ func NewLevelDBStore(cfg LevelDBOptions) (*LevelDBStore, error) {
}, nil }, nil
} }
// Put implements the Store interface.
func (s *LevelDBStore) Put(key, value []byte) error {
return s.db.Put(key, value, nil)
}
// Get implements the Store interface. // Get implements the Store interface.
func (s *LevelDBStore) Get(key []byte) ([]byte, error) { func (s *LevelDBStore) Get(key []byte) ([]byte, error) {
value, err := s.db.Get(key, nil) value, err := s.db.Get(key, nil)
@ -50,26 +45,16 @@ func (s *LevelDBStore) Get(key []byte) ([]byte, error) {
return value, err return value, err
} }
// Delete implements the Store interface.
func (s *LevelDBStore) Delete(key []byte) error {
return s.db.Delete(key, nil)
}
// PutBatch implements the Store interface.
func (s *LevelDBStore) PutBatch(batch Batch) error {
lvldbBatch := batch.(*leveldb.Batch)
return s.db.Write(lvldbBatch, nil)
}
// PutChangeSet implements the Store interface. // PutChangeSet implements the Store interface.
func (s *LevelDBStore) PutChangeSet(puts map[string][]byte) error { func (s *LevelDBStore) PutChangeSet(puts map[string][]byte, stores map[string][]byte) error {
tx, err := s.db.OpenTransaction() tx, err := s.db.OpenTransaction()
if err != nil { if err != nil {
return err return err
} }
for k := range puts { for _, m := range []map[string][]byte{puts, stores} {
if puts[k] != nil { for k := range m {
err = tx.Put([]byte(k), puts[k], nil) if m[k] != nil {
err = tx.Put([]byte(k), m[k], nil)
} else { } else {
err = tx.Delete([]byte(k), nil) err = tx.Delete([]byte(k), nil)
} }
@ -78,6 +63,7 @@ func (s *LevelDBStore) PutChangeSet(puts map[string][]byte) error {
return err return err
} }
} }
}
return tx.Commit() return tx.Commit()
} }
@ -131,12 +117,6 @@ func (s *LevelDBStore) seek(iter iterator.Iterator, backwards bool, f func(k, v
iter.Release() iter.Release()
} }
// Batch implements the Batch interface and returns a leveldb
// compatible Batch.
func (s *LevelDBStore) Batch() Batch {
return new(leveldb.Batch)
}
// Close implements the Store interface. // Close implements the Store interface.
func (s *LevelDBStore) Close() error { func (s *LevelDBStore) Close() error {
return s.db.Close() return s.db.Close()

126
pkg/core/storage/memcached_store.go
View file

@ -15,6 +15,7 @@ import (
type MemCachedStore struct { type MemCachedStore struct {
MemoryStore MemoryStore
private bool
// plock protects Persist from double entrance. // plock protects Persist from double entrance.
plock sync.Mutex plock sync.Mutex
// Persistent Store. // Persistent Store.
@ -51,11 +52,50 @@ func NewMemCachedStore(lower Store) *MemCachedStore {
} }
} }
// NewPrivateMemCachedStore creates a new private (unlocked) MemCachedStore object.
// Private cached stores are closed after Persist.
func NewPrivateMemCachedStore(lower Store) *MemCachedStore {
return &MemCachedStore{
MemoryStore: *NewMemoryStore(),
private: true,
ps: lower,
}
}
// lock write-locks non-private store.
func (s *MemCachedStore) lock() {
if !s.private {
s.mut.Lock()
}
}
// unlock unlocks non-private store.
func (s *MemCachedStore) unlock() {
if !s.private {
s.mut.Unlock()
}
}
// rlock read-locks non-private store.
func (s *MemCachedStore) rlock() {
if !s.private {
s.mut.RLock()
}
}
// runlock drops read lock for non-private stores.
func (s *MemCachedStore) runlock() {
if !s.private {
s.mut.RUnlock()
}
}
// Get implements the Store interface. // Get implements the Store interface.
func (s *MemCachedStore) Get(key []byte) ([]byte, error) { func (s *MemCachedStore) Get(key []byte) ([]byte, error) {
s.mut.RLock() s.rlock()
defer s.mut.RUnlock() defer s.runlock()
if val, ok := s.mem[string(key)]; ok { m := s.chooseMap(key)
if val, ok := m[string(key)]; ok {
if val == nil { if val == nil {
return nil, ErrKeyNotFound return nil, ErrKeyNotFound
} }
@ -64,16 +104,33 @@ func (s *MemCachedStore) Get(key []byte) ([]byte, error) {
return s.ps.Get(key) return s.ps.Get(key)
} }
// Put puts new KV pair into the store.
func (s *MemCachedStore) Put(key, value []byte) {
newKey := string(key)
vcopy := slice.Copy(value)
s.lock()
put(s.chooseMap(key), newKey, vcopy)
s.unlock()
}
// Delete drops KV pair from the store. Never returns an error.
func (s *MemCachedStore) Delete(key []byte) {
newKey := string(key)
s.lock()
put(s.chooseMap(key), newKey, nil)
s.unlock()
}
// GetBatch returns currently accumulated changeset. // GetBatch returns currently accumulated changeset.
func (s *MemCachedStore) GetBatch() *MemBatch { func (s *MemCachedStore) GetBatch() *MemBatch {
s.mut.RLock() s.rlock()
defer s.mut.RUnlock() defer s.runlock()
var b MemBatch var b MemBatch
b.Put = make([]KeyValueExists, 0, len(s.mem)) b.Put = make([]KeyValueExists, 0, len(s.mem)+len(s.stor))
b.Deleted = make([]KeyValueExists, 0) b.Deleted = make([]KeyValueExists, 0)
for k, v := range s.mem { for _, m := range []map[string][]byte{s.mem, s.stor} {
for k, v := range m {
key := []byte(k) key := []byte(k)
_, err := s.ps.Get(key) _, err := s.ps.Get(key)
if v == nil { if v == nil {
@ -82,14 +139,32 @@ func (s *MemCachedStore) GetBatch() *MemBatch {
b.Put = append(b.Put, KeyValueExists{KeyValue: KeyValue{Key: key, Value: v}, Exists: err == nil}) b.Put = append(b.Put, KeyValueExists{KeyValue: KeyValue{Key: key, Value: v}, Exists: err == nil})
} }
} }
}
return &b return &b
} }
// PutChangeSet implements the Store interface. Never returns an error.
func (s *MemCachedStore) PutChangeSet(puts map[string][]byte, stores map[string][]byte) error {
s.lock()
s.MemoryStore.putChangeSet(puts, stores)
s.unlock()
return nil
}
// Seek implements the Store interface. // Seek implements the Store interface.
func (s *MemCachedStore) Seek(rng SeekRange, f func(k, v []byte) bool) { func (s *MemCachedStore) Seek(rng SeekRange, f func(k, v []byte) bool) {
s.seek(context.Background(), rng, false, f) s.seek(context.Background(), rng, false, f)
} }
// GetStorageChanges returns all current storage changes. It can only be done for private
// MemCachedStore.
func (s *MemCachedStore) GetStorageChanges() map[string][]byte {
if !s.private {
panic("GetStorageChanges called on shared MemCachedStore")
}
return s.stor
}
// SeekAsync returns non-buffered channel with matching KeyValue pairs. Key and // SeekAsync returns non-buffered channel with matching KeyValue pairs. Key and
// value slices may not be copied and may be modified. SeekAsync can guarantee // value slices may not be copied and may be modified. SeekAsync can guarantee
// that key-value items are sorted by key in ascending way. // that key-value items are sorted by key in ascending way.
@ -130,8 +205,9 @@ func (s *MemCachedStore) seek(ctx context.Context, rng SeekRange, cutPrefix bool
return strings.HasPrefix(key, sPrefix) && (lStart == 0 || strings.Compare(key[lPrefix:], sStart) <= 0) return strings.HasPrefix(key, sPrefix) && (lStart == 0 || strings.Compare(key[lPrefix:], sStart) <= 0)
} }
} }
s.mut.RLock() s.rlock()
for k, v := range s.MemoryStore.mem { m := s.MemoryStore.chooseMap(rng.Prefix)
for k, v := range m {
if isKeyOK(k) { if isKeyOK(k) {
memRes = append(memRes, KeyValueExists{ memRes = append(memRes, KeyValueExists{
KeyValue: KeyValue{ KeyValue: KeyValue{
@ -143,8 +219,7 @@ func (s *MemCachedStore) seek(ctx context.Context, rng SeekRange, cutPrefix bool
} }
} }
ps := s.ps ps := s.ps
s.mut.RUnlock() s.runlock()
less := func(k1, k2 []byte) bool { less := func(k1, k2 []byte) bool {
res := bytes.Compare(k1, k2) res := bytes.Compare(k1, k2)
return res != 0 && rng.Backwards == (res > 0) return res != 0 && rng.Backwards == (res > 0)
@ -255,11 +330,25 @@ func (s *MemCachedStore) persist(isSync bool) (int, error) {
var err error var err error
var keys int var keys int
if s.private {
keys = len(s.mem) + len(s.stor)
if keys == 0 {
return 0, nil
}
err = s.ps.PutChangeSet(s.mem, s.stor)
if err != nil {
return 0, err
}
s.mem = nil
s.stor = nil
return keys, nil
}
s.plock.Lock() s.plock.Lock()
defer s.plock.Unlock() defer s.plock.Unlock()
s.mut.Lock() s.mut.Lock()
keys = len(s.mem) keys = len(s.mem) + len(s.stor)
if keys == 0 { if keys == 0 {
s.mut.Unlock() s.mut.Unlock()
return 0, nil return 0, nil
@ -269,14 +358,15 @@ func (s *MemCachedStore) persist(isSync bool) (int, error) {
// starts using fresh new maps. This tempstore is only known here and // starts using fresh new maps. This tempstore is only known here and
// nothing ever changes it, therefore accesses to it (reads) can go // nothing ever changes it, therefore accesses to it (reads) can go
// unprotected while writes are handled by s proper. // unprotected while writes are handled by s proper.
var tempstore = &MemCachedStore{MemoryStore: MemoryStore{mem: s.mem}, ps: s.ps} var tempstore = &MemCachedStore{MemoryStore: MemoryStore{mem: s.mem, stor: s.stor}, ps: s.ps}
s.ps = tempstore s.ps = tempstore
s.mem = make(map[string][]byte, len(s.mem)) s.mem = make(map[string][]byte, len(s.mem))
s.stor = make(map[string][]byte, len(s.stor))
if !isSync { if !isSync {
s.mut.Unlock() s.mut.Unlock()
} }
err = tempstore.ps.PutChangeSet(tempstore.mem) err = tempstore.ps.PutChangeSet(tempstore.mem, tempstore.stor)
if !isSync { if !isSync {
s.mut.Lock() s.mut.Lock()
@ -290,10 +380,14 @@ func (s *MemCachedStore) persist(isSync bool) (int, error) {
// We're toast. We'll try to still keep proper state, but OOM // We're toast. We'll try to still keep proper state, but OOM
// killer will get to us eventually. // killer will get to us eventually.
for k := range s.mem { for k := range s.mem {
tempstore.put(k, s.mem[k]) put(tempstore.mem, k, s.mem[k])
}
for k := range s.stor {
put(tempstore.stor, k, s.stor[k])
} }
s.ps = tempstore.ps s.ps = tempstore.ps
s.mem = tempstore.mem s.mem = tempstore.mem
s.stor = tempstore.stor
} }
s.mut.Unlock() s.mut.Unlock()
return keys, err return keys, err

141
pkg/core/storage/memcached_store_test.go
View file

@ -12,6 +12,40 @@ import (
"github.com/stretchr/testify/require" "github.com/stretchr/testify/require"
) )
func TestMemCachedPutGetDelete(t *testing.T) {
ps := NewMemoryStore()
s := NewMemCachedStore(ps)
key := []byte("foo")
value := []byte("bar")
s.Put(key, value)
result, err := s.Get(key)
assert.Nil(t, err)
require.Equal(t, value, result)
s.Delete(key)
_, err = s.Get(key)
assert.NotNil(t, err)
assert.Equal(t, err, ErrKeyNotFound)
// Double delete.
s.Delete(key)
_, err = s.Get(key)
assert.NotNil(t, err)
assert.Equal(t, err, ErrKeyNotFound)
// Nonexistent.
key = []byte("sparse")
s.Delete(key)
_, err = s.Get(key)
assert.NotNil(t, err)
assert.Equal(t, err, ErrKeyNotFound)
}
func testMemCachedStorePersist(t *testing.T, ps Store) { func testMemCachedStorePersist(t *testing.T, ps Store) {
// cached Store // cached Store
ts := NewMemCachedStore(ps) ts := NewMemCachedStore(ps)
@ -20,7 +54,7 @@ func testMemCachedStorePersist(t *testing.T, ps Store) {
assert.Equal(t, nil, err) assert.Equal(t, nil, err)
assert.Equal(t, 0, c) assert.Equal(t, 0, c)
// persisting one key should result in one key in ps and nothing in ts // persisting one key should result in one key in ps and nothing in ts
assert.NoError(t, ts.Put([]byte("key"), []byte("value"))) ts.Put([]byte("key"), []byte("value"))
checkBatch(t, ts, []KeyValueExists{{KeyValue: KeyValue{Key: []byte("key"), Value: []byte("value")}}}, nil) checkBatch(t, ts, []KeyValueExists{{KeyValue: KeyValue{Key: []byte("key"), Value: []byte("value")}}}, nil)
c, err = ts.Persist() c, err = ts.Persist()
checkBatch(t, ts, nil, nil) checkBatch(t, ts, nil, nil)
@ -33,8 +67,8 @@ func testMemCachedStorePersist(t *testing.T, ps Store) {
assert.Equal(t, ErrKeyNotFound, err) assert.Equal(t, ErrKeyNotFound, err)
assert.Equal(t, []byte(nil), v) assert.Equal(t, []byte(nil), v)
// now we overwrite the previous `key` contents and also add `key2`, // now we overwrite the previous `key` contents and also add `key2`,
assert.NoError(t, ts.Put([]byte("key"), []byte("newvalue"))) ts.Put([]byte("key"), []byte("newvalue"))
assert.NoError(t, ts.Put([]byte("key2"), []byte("value2"))) ts.Put([]byte("key2"), []byte("value2"))
// this is to check that now key is written into the ps before we do // this is to check that now key is written into the ps before we do
// persist // persist
v, err = ps.Get([]byte("key2")) v, err = ps.Get([]byte("key2"))
@ -68,8 +102,7 @@ func testMemCachedStorePersist(t *testing.T, ps Store) {
assert.Equal(t, nil, err) assert.Equal(t, nil, err)
assert.Equal(t, 0, c) assert.Equal(t, 0, c)
// test persisting deletions // test persisting deletions
err = ts.Delete([]byte("key")) ts.Delete([]byte("key"))
assert.Equal(t, nil, err)
checkBatch(t, ts, nil, []KeyValueExists{{KeyValue: KeyValue{Key: []byte("key")}, Exists: true}}) checkBatch(t, ts, nil, []KeyValueExists{{KeyValue: KeyValue{Key: []byte("key")}, Exists: true}})
c, err = ts.Persist() c, err = ts.Persist()
checkBatch(t, ts, nil, nil) checkBatch(t, ts, nil, nil)
@ -123,11 +156,11 @@ func TestCachedGetFromPersistent(t *testing.T) {
ps := NewMemoryStore() ps := NewMemoryStore()
ts := NewMemCachedStore(ps) ts := NewMemCachedStore(ps)
assert.NoError(t, ps.Put(key, value)) assert.NoError(t, ps.PutChangeSet(map[string][]byte{string(key): value}, nil))
val, err := ts.Get(key) val, err := ts.Get(key)
assert.Nil(t, err) assert.Nil(t, err)
assert.Equal(t, value, val) assert.Equal(t, value, val)
assert.NoError(t, ts.Delete(key)) ts.Delete(key)
val, err = ts.Get(key) val, err = ts.Get(key)
assert.Equal(t, err, ErrKeyNotFound) assert.Equal(t, err, ErrKeyNotFound)
assert.Nil(t, val) assert.Nil(t, val)
@ -156,15 +189,15 @@ func TestCachedSeek(t *testing.T) {
ts = NewMemCachedStore(ps) ts = NewMemCachedStore(ps)
) )
for _, v := range lowerKVs { for _, v := range lowerKVs {
require.NoError(t, ps.Put(v.Key, v.Value)) require.NoError(t, ps.PutChangeSet(map[string][]byte{string(v.Key): v.Value}, nil))
} }
for _, v := range deletedKVs { for _, v := range deletedKVs {
require.NoError(t, ps.Put(v.Key, v.Value)) require.NoError(t, ps.PutChangeSet(map[string][]byte{string(v.Key): v.Value}, nil))
require.NoError(t, ts.Delete(v.Key)) ts.Delete(v.Key)
} }
for _, v := range updatedKVs { for _, v := range updatedKVs {
require.NoError(t, ps.Put(v.Key, []byte("stub"))) require.NoError(t, ps.PutChangeSet(map[string][]byte{string(v.Key): v.Value}, nil))
require.NoError(t, ts.Put(v.Key, v.Value)) ts.Put(v.Key, v.Value)
} }
foundKVs := make(map[string][]byte) foundKVs := make(map[string][]byte)
ts.Seek(SeekRange{Prefix: goodPrefix}, func(k, v []byte) bool { ts.Seek(SeekRange{Prefix: goodPrefix}, func(k, v []byte) bool {
@ -199,36 +232,38 @@ func benchmarkCachedSeek(t *testing.B, ps Store, psElementsCount, tsElementsCoun
) )
for i := 0; i < psElementsCount; i++ { for i := 0; i < psElementsCount; i++ {
// lower KVs with matching prefix that should be found // lower KVs with matching prefix that should be found
require.NoError(t, ps.Put(append(lowerPrefixGood, random.Bytes(10)...), []byte("value"))) ts.Put(append(lowerPrefixGood, random.Bytes(10)...), []byte("value"))
// lower KVs with non-matching prefix that shouldn't be found // lower KVs with non-matching prefix that shouldn't be found
require.NoError(t, ps.Put(append(lowerPrefixBad, random.Bytes(10)...), []byte("value"))) ts.Put(append(lowerPrefixBad, random.Bytes(10)...), []byte("value"))
// deleted KVs with matching prefix that shouldn't be found // deleted KVs with matching prefix that shouldn't be found
key := append(deletedPrefixGood, random.Bytes(10)...) key := append(deletedPrefixGood, random.Bytes(10)...)
require.NoError(t, ps.Put(key, []byte("deleted"))) ts.Put(key, []byte("deleted"))
if i < tsElementsCount { if i < tsElementsCount {
require.NoError(t, ts.Delete(key)) ts.Delete(key)
} }
// deleted KVs with non-matching prefix that shouldn't be found // deleted KVs with non-matching prefix that shouldn't be found
key = append(deletedPrefixBad, random.Bytes(10)...) key = append(deletedPrefixBad, random.Bytes(10)...)
require.NoError(t, ps.Put(key, []byte("deleted"))) ts.Put(key, []byte("deleted"))
if i < tsElementsCount { if i < tsElementsCount {
require.NoError(t, ts.Delete(key)) ts.Delete(key)
} }
// updated KVs with matching prefix that should be found // updated KVs with matching prefix that should be found
key = append(updatedPrefixGood, random.Bytes(10)...) key = append(updatedPrefixGood, random.Bytes(10)...)
require.NoError(t, ps.Put(key, []byte("stub"))) ts.Put(key, []byte("stub"))
if i < tsElementsCount { if i < tsElementsCount {
require.NoError(t, ts.Put(key, []byte("updated"))) ts.Put(key, []byte("updated"))
} }
// updated KVs with non-matching prefix that shouldn't be found // updated KVs with non-matching prefix that shouldn't be found
key = append(updatedPrefixBad, random.Bytes(10)...) key = append(updatedPrefixBad, random.Bytes(10)...)
require.NoError(t, ps.Put(key, []byte("stub"))) ts.Put(key, []byte("stub"))
if i < tsElementsCount { if i < tsElementsCount {
require.NoError(t, ts.Put(key, []byte("updated"))) ts.Put(key, []byte("updated"))
} }
} }
_, err := ts.PersistSync()
require.NoError(t, err)
t.ReportAllocs() t.ReportAllocs()
t.ResetTimer() t.ResetTimer()
@ -263,18 +298,10 @@ func newMemCachedStoreForTesting(t testing.TB) Store {
return NewMemCachedStore(NewMemoryStore()) return NewMemCachedStore(NewMemoryStore())
} }
type BadBatch struct{}
func (b BadBatch) Delete(k []byte) {}
func (b BadBatch) Put(k, v []byte) {}
type BadStore struct { type BadStore struct {
onPutBatch func() onPutBatch func()
} }
func (b *BadStore) Batch() Batch {
return BadBatch{}
}
func (b *BadStore) Delete(k []byte) error { func (b *BadStore) Delete(k []byte) error {
return nil return nil
} }
@ -284,10 +311,7 @@ func (b *BadStore) Get([]byte) ([]byte, error) {
func (b *BadStore) Put(k, v []byte) error { func (b *BadStore) Put(k, v []byte) error {
return nil return nil
} }
func (b *BadStore) PutBatch(Batch) error { func (b *BadStore) PutChangeSet(_ map[string][]byte, _ map[string][]byte) error {
return nil
}
func (b *BadStore) PutChangeSet(_ map[string][]byte) error {
b.onPutBatch() b.onPutBatch()
return ErrKeyNotFound return ErrKeyNotFound
} }
@ -310,13 +334,13 @@ func TestMemCachedPersistFailing(t *testing.T) {
// cached Store // cached Store
ts := NewMemCachedStore(&bs) ts := NewMemCachedStore(&bs)
// Set a pair of keys. // Set a pair of keys.
require.NoError(t, ts.Put(t1, t1)) ts.Put(t1, t1)
require.NoError(t, ts.Put(t2, t2)) ts.Put(t2, t2)
// This will be called during Persist(). // This will be called during Persist().
bs.onPutBatch = func() { bs.onPutBatch = func() {
// Drop one, add one. // Drop one, add one.
require.NoError(t, ts.Put(b1, b1)) ts.Put(b1, b1)
require.NoError(t, ts.Delete(t1)) ts.Delete(t1)
} }
_, err := ts.Persist() _, err := ts.Persist()
require.Error(t, err) require.Error(t, err)
@ -331,6 +355,31 @@ func TestMemCachedPersistFailing(t *testing.T) {
require.Equal(t, b1, res) require.Equal(t, b1, res)
} }
func TestPrivateMemCachedPersistFailing(t *testing.T) {
var (
bs BadStore
t1 = []byte("t1")
t2 = []byte("t2")
)
// cached Store
ts := NewPrivateMemCachedStore(&bs)
// Set a pair of keys.
ts.Put(t1, t1)
ts.Put(t2, t2)
// This will be called during Persist().
bs.onPutBatch = func() {}
_, err := ts.Persist()
require.Error(t, err)
// PutBatch() failed in Persist, but we still should have proper state.
res, err := ts.Get(t1)
require.NoError(t, err)
require.Equal(t, t1, res)
res, err = ts.Get(t2)
require.NoError(t, err)
require.Equal(t, t2, res)
}
func TestCachedSeekSorting(t *testing.T) { func TestCachedSeekSorting(t *testing.T) {
var ( var (
// Given this prefix... // Given this prefix...
@ -354,19 +403,20 @@ func TestCachedSeekSorting(t *testing.T) {
{[]byte{1, 3, 2}, []byte("wop")}, {[]byte{1, 3, 2}, []byte("wop")},
{[]byte{1, 3, 4}, []byte("zaq")}, {[]byte{1, 3, 4}, []byte("zaq")},
} }
ps = NewMemoryStore()
ts = NewMemCachedStore(ps)
) )
for _, newCached := range []func(Store) *MemCachedStore{NewMemCachedStore, NewPrivateMemCachedStore} {
ps := NewMemoryStore()
ts := newCached(ps)
for _, v := range lowerKVs { for _, v := range lowerKVs {
require.NoError(t, ps.Put(v.Key, v.Value)) require.NoError(t, ps.PutChangeSet(map[string][]byte{string(v.Key): v.Value}, nil))
} }
for _, v := range deletedKVs { for _, v := range deletedKVs {
require.NoError(t, ps.Put(v.Key, v.Value)) require.NoError(t, ps.PutChangeSet(map[string][]byte{string(v.Key): v.Value}, nil))
require.NoError(t, ts.Delete(v.Key)) ts.Delete(v.Key)
} }
for _, v := range updatedKVs { for _, v := range updatedKVs {
require.NoError(t, ps.Put(v.Key, []byte("stub"))) require.NoError(t, ps.PutChangeSet(map[string][]byte{string(v.Key): v.Value}, nil))
require.NoError(t, ts.Put(v.Key, v.Value)) ts.Put(v.Key, v.Value)
} }
var foundKVs []KeyValue var foundKVs []KeyValue
ts.Seek(SeekRange{Prefix: goodPrefix}, func(k, v []byte) bool { ts.Seek(SeekRange{Prefix: goodPrefix}, func(k, v []byte) bool {
@ -380,3 +430,4 @@ func TestCachedSeekSorting(t *testing.T) {
}) })
require.Equal(t, expected, foundKVs) require.Equal(t, expected, foundKVs)
} }
}

109
pkg/core/storage/memory_store.go
View file

@ -5,8 +5,6 @@ import (
"sort" "sort"
"strings" "strings"
"sync" "sync"
"github.com/nspcc-dev/neo-go/pkg/util/slice"
) )
// MemoryStore is an in-memory implementation of a Store, mainly // MemoryStore is an in-memory implementation of a Store, mainly
@ -14,27 +12,14 @@ import (
type MemoryStore struct { type MemoryStore struct {
mut sync.RWMutex mut sync.RWMutex
mem map[string][]byte mem map[string][]byte
} stor map[string][]byte
// MemoryBatch is an in-memory batch compatible with MemoryStore.
type MemoryBatch struct {
MemoryStore
}
// Put implements the Batch interface.
func (b *MemoryBatch) Put(k, v []byte) {
b.MemoryStore.put(string(k), slice.Copy(v))
}
// Delete implements Batch interface.
func (b *MemoryBatch) Delete(k []byte) {
b.MemoryStore.drop(string(k))
} }
// NewMemoryStore creates a new MemoryStore object. // NewMemoryStore creates a new MemoryStore object.
func NewMemoryStore() *MemoryStore { func NewMemoryStore() *MemoryStore {
return &MemoryStore{ return &MemoryStore{
mem: make(map[string][]byte), mem: make(map[string][]byte),
stor: make(map[string][]byte),
} }
} }
@ -42,59 +27,45 @@ func NewMemoryStore() *MemoryStore {
func (s *MemoryStore) Get(key []byte) ([]byte, error) { func (s *MemoryStore) Get(key []byte) ([]byte, error) {
s.mut.RLock() s.mut.RLock()
defer s.mut.RUnlock() defer s.mut.RUnlock()
if val, ok := s.mem[string(key)]; ok && val != nil { m := s.chooseMap(key)
if val, ok := m[string(key)]; ok && val != nil {
return val, nil return val, nil
} }
return nil, ErrKeyNotFound return nil, ErrKeyNotFound
} }
func (s *MemoryStore) chooseMap(key []byte) map[string][]byte {
switch KeyPrefix(key[0]) {
case STStorage, STTempStorage:
return s.stor
default:
return s.mem
}
}
// put puts a key-value pair into the store, it's supposed to be called // put puts a key-value pair into the store, it's supposed to be called
// with mutex locked. // with mutex locked.
func (s *MemoryStore) put(key string, value []byte) { func put(m map[string][]byte, key string, value []byte) {
s.mem[key] = value m[key] = value
}
// Put implements the Store interface. Never returns an error.
func (s *MemoryStore) Put(key, value []byte) error {
newKey := string(key)
vcopy := slice.Copy(value)
s.mut.Lock()
s.put(newKey, vcopy)
s.mut.Unlock()
return nil
}
// drop deletes a key-value pair from the store, it's supposed to be called
// with mutex locked.
func (s *MemoryStore) drop(key string) {
s.mem[key] = nil
}
// Delete implements Store interface. Never returns an error.
func (s *MemoryStore) Delete(key []byte) error {
newKey := string(key)
s.mut.Lock()
s.drop(newKey)
s.mut.Unlock()
return nil
}
// PutBatch implements the Store interface. Never returns an error.
func (s *MemoryStore) PutBatch(batch Batch) error {
b := batch.(*MemoryBatch)
return s.PutChangeSet(b.mem)
} }
// PutChangeSet implements the Store interface. Never returns an error. // PutChangeSet implements the Store interface. Never returns an error.
func (s *MemoryStore) PutChangeSet(puts map[string][]byte) error { func (s *MemoryStore) PutChangeSet(puts map[string][]byte, stores map[string][]byte) error {
s.mut.Lock() s.mut.Lock()
for k := range puts { s.putChangeSet(puts, stores)
s.put(k, puts[k])
}
s.mut.Unlock() s.mut.Unlock()
return nil return nil
} }
func (s *MemoryStore) putChangeSet(puts map[string][]byte, stores map[string][]byte) {
for k := range puts {
put(s.mem, k, puts[k])
}
for k := range stores {
put(s.stor, k, stores[k])
}
}
// Seek implements the Store interface. // Seek implements the Store interface.
func (s *MemoryStore) Seek(rng SeekRange, f func(k, v []byte) bool) { func (s *MemoryStore) Seek(rng SeekRange, f func(k, v []byte) bool) {
s.mut.RLock() s.mut.RLock()
@ -109,7 +80,7 @@ func (s *MemoryStore) SeekGC(rng SeekRange, keep func(k, v []byte) bool) error {
// sensitive to the order of KV pairs. // sensitive to the order of KV pairs.
s.seek(rng, func(k, v []byte) bool { s.seek(rng, func(k, v []byte) bool {
if !keep(k, v) { if !keep(k, v) {
s.drop(string(k)) delete(s.chooseMap(k), string(k))
} }
return true return true
}) })
@ -117,18 +88,6 @@ func (s *MemoryStore) SeekGC(rng SeekRange, keep func(k, v []byte) bool) error {
return nil return nil
} }
// SeekAll is like seek but also iterates over deleted items.
func (s *MemoryStore) SeekAll(key []byte, f func(k, v []byte)) {
s.mut.RLock()
defer s.mut.RUnlock()
sk := string(key)
for k, v := range s.mem {
if strings.HasPrefix(k, sk) {
f([]byte(k), v)
}
}
}
// seek is an internal unlocked implementation of Seek. `start` denotes whether // seek is an internal unlocked implementation of Seek. `start` denotes whether
// seeking starting from the provided prefix should be performed. Backwards // seeking starting from the provided prefix should be performed. Backwards
// seeking from some point is supported with corresponding SeekRange field set. // seeking from some point is supported with corresponding SeekRange field set.
@ -152,7 +111,8 @@ func (s *MemoryStore) seek(rng SeekRange, f func(k, v []byte) bool) {
return res != 0 && rng.Backwards == (res > 0) return res != 0 && rng.Backwards == (res > 0)
} }
for k, v := range s.mem { m := s.chooseMap(rng.Prefix)
for k, v := range m {
if v != nil && isKeyOK(k) { if v != nil && isKeyOK(k) {
memList = append(memList, KeyValue{ memList = append(memList, KeyValue{
Key: []byte(k), Key: []byte(k),
@ -170,21 +130,12 @@ func (s *MemoryStore) seek(rng SeekRange, f func(k, v []byte) bool) {
} }
} }
// Batch implements the Batch interface and returns a compatible Batch.
func (s *MemoryStore) Batch() Batch {
return newMemoryBatch()
}
// newMemoryBatch returns new memory batch.
func newMemoryBatch() *MemoryBatch {
return &MemoryBatch{MemoryStore: *NewMemoryStore()}
}
// Close implements Store interface and clears up memory. Never returns an // Close implements Store interface and clears up memory. Never returns an
// error. // error.
func (s *MemoryStore) Close() error { func (s *MemoryStore) Close() error {
s.mut.Lock() s.mut.Lock()
s.mem = nil s.mem = nil
s.stor = nil
s.mut.Unlock() s.mut.Unlock()
return nil return nil
} }

7
pkg/core/storage/memory_store_test.go
View file

@ -20,10 +20,13 @@ func BenchmarkMemorySeek(t *testing.B) {
searchPrefix = []byte{1} searchPrefix = []byte{1}
badPrefix = []byte{2} badPrefix = []byte{2}
) )
ts := NewMemCachedStore(ms)
for i := 0; i < count; i++ { for i := 0; i < count; i++ {
require.NoError(t, ms.Put(append(searchPrefix, random.Bytes(10)...), random.Bytes(10))) ts.Put(append(searchPrefix, random.Bytes(10)...), random.Bytes(10))
require.NoError(t, ms.Put(append(badPrefix, random.Bytes(10)...), random.Bytes(10))) ts.Put(append(badPrefix, random.Bytes(10)...), random.Bytes(10))
} }
_, err := ts.PersistSync()
require.NoError(t, err)
t.ReportAllocs() t.ReportAllocs()
t.ResetTimer() t.ResetTimer()

23
pkg/core/storage/store.go
View file

@ -61,8 +61,7 @@ type Operation struct {
// SeekRange represents options for Store.Seek operation. // SeekRange represents options for Store.Seek operation.
type SeekRange struct { type SeekRange struct {
// Prefix denotes the Seek's lookup key. // Prefix denotes the Seek's lookup key.
// Empty Prefix means seeking through all keys in the DB starting from // Empty Prefix is not supported.
// the Start if specified.
Prefix []byte Prefix []byte
// Start denotes value appended to the Prefix to start Seek from. // Start denotes value appended to the Prefix to start Seek from.
// Seeking starting from some key includes this key to the result; // Seeking starting from some key includes this key to the result;
@ -83,16 +82,13 @@ type SeekRange struct {
var ErrKeyNotFound = errors.New("key not found") var ErrKeyNotFound = errors.New("key not found")
type ( type (
// Store is anything that can persist and retrieve the blockchain. // Store is the underlying KV backend for the blockchain data, it's
// information. // not intended to be used directly, you wrap it with some memory cache
// layer most of the time.
Store interface { Store interface {
Batch() Batch
Delete(k []byte) error
Get([]byte) ([]byte, error) Get([]byte) ([]byte, error)
Put(k, v []byte) error
PutBatch(Batch) error
// PutChangeSet allows to push prepared changeset to the Store. // PutChangeSet allows to push prepared changeset to the Store.
PutChangeSet(puts map[string][]byte) error PutChangeSet(puts map[string][]byte, stor map[string][]byte) error
// Seek can guarantee that provided key (k) and value (v) are the only valid until the next call to f. // Seek can guarantee that provided key (k) and value (v) are the only valid until the next call to f.
// Seek continues iteration until false is returned from f. // Seek continues iteration until false is returned from f.
// Key and value slices should not be modified. // Key and value slices should not be modified.
@ -106,15 +102,6 @@ type (
Close() error Close() error
} }
// Batch represents an abstraction on top of batch operations.
// Each Store implementation is responsible of casting a Batch
// to its appropriate type. Batches can only be used in a single
// thread.
Batch interface {
Delete(k []byte)
Put(k, v []byte)
}
// KeyPrefix is a constant byte added as a prefix for each key // KeyPrefix is a constant byte added as a prefix for each key
// stored. // stored.
KeyPrefix uint8 KeyPrefix uint8

256
pkg/core/storage/storeandbatch_test.go
View file

@ -19,17 +19,6 @@ type dbSetup struct {
type dbTestFunction func(*testing.T, Store) type dbTestFunction func(*testing.T, Store)
func testStorePutAndGet(t *testing.T, s Store) {
key := []byte("foo")
value := []byte("bar")
require.NoError(t, s.Put(key, value))
result, err := s.Get(key)
assert.Nil(t, err)
require.Equal(t, value, result)
}
func testStoreGetNonExistent(t *testing.T, s Store) { func testStoreGetNonExistent(t *testing.T, s Store) {
key := []byte("sparse") key := []byte("sparse")
@ -37,28 +26,7 @@ func testStoreGetNonExistent(t *testing.T, s Store) {
assert.Equal(t, err, ErrKeyNotFound) assert.Equal(t, err, ErrKeyNotFound)
} }
func testStorePutBatch(t *testing.T, s Store) { func pushSeekDataSet(t *testing.T, s Store) []KeyValue {
var (
key = []byte("foo")
value = []byte("bar")
batch = s.Batch()
)
// Test that key and value are copied when batching.
keycopy := slice.Copy(key)
valuecopy := slice.Copy(value)
batch.Put(keycopy, valuecopy)
copy(valuecopy, key)
copy(keycopy, value)
require.NoError(t, s.PutBatch(batch))
newVal, err := s.Get(key)
assert.Nil(t, err)
require.Equal(t, value, newVal)
assert.Equal(t, value, newVal)
}
func testStoreSeek(t *testing.T, s Store) {
// Use the same set of kvs to test Seek with different prefix/start values. // Use the same set of kvs to test Seek with different prefix/start values.
kvs := []KeyValue{ kvs := []KeyValue{
{[]byte("10"), []byte("bar")}, {[]byte("10"), []byte("bar")},
@ -69,10 +37,17 @@ func testStoreSeek(t *testing.T, s Store) {
{[]byte("30"), []byte("bare")}, {[]byte("30"), []byte("bare")},
{[]byte("31"), []byte("barf")}, {[]byte("31"), []byte("barf")},
} }
up := NewMemCachedStore(s)
for _, v := range kvs { for _, v := range kvs {
require.NoError(t, s.Put(v.Key, v.Value)) up.Put(v.Key, v.Value)
}
_, err := up.PersistSync()
require.NoError(t, err)
return kvs
} }
func testStoreSeek(t *testing.T, s Store) {
kvs := pushSeekDataSet(t, s)
check := func(t *testing.T, goodprefix, start []byte, goodkvs []KeyValue, backwards bool, cont func(k, v []byte) bool) { check := func(t *testing.T, goodprefix, start []byte, goodkvs []KeyValue, backwards bool, cont func(k, v []byte) bool) {
// Seek result expected to be sorted in an ascending (for forwards seeking) or descending (for backwards seeking) way. // Seek result expected to be sorted in an ascending (for forwards seeking) or descending (for backwards seeking) way.
cmpFunc := func(i, j int) bool { cmpFunc := func(i, j int) bool {
@ -228,215 +203,10 @@ func testStoreSeek(t *testing.T, s Store) {
}) })
}) })
}) })
t.Run("empty prefix, non-empty start", func(t *testing.T) {
t.Run("forwards", func(t *testing.T) {
t.Run("good", func(t *testing.T) {
goodprefix := []byte{}
start := []byte("21")
goodkvs := []KeyValue{
kvs[3], // key = "21"
kvs[4], // key = "22"
kvs[5], // key = "30"
kvs[6], // key = "31"
}
check(t, goodprefix, start, goodkvs, false, nil)
})
t.Run("no matching items", func(t *testing.T) {
goodprefix := []byte{}
start := []byte("32") // start is more than all keys.
check(t, goodprefix, start, []KeyValue{}, false, nil)
})
t.Run("early stop", func(t *testing.T) {
goodprefix := []byte{}
start := []byte("21")
goodkvs := []KeyValue{
kvs[3], // key = "21"
kvs[4], // key = "22"
kvs[5], // key = "30"
}
check(t, goodprefix, start, goodkvs, false, func(k, v []byte) bool {
return string(k) < "30"
})
})
})
t.Run("backwards", func(t *testing.T) {
t.Run("good", func(t *testing.T) {
goodprefix := []byte{}
start := []byte("21")
goodkvs := []KeyValue{
kvs[3], // key = "21"
kvs[2], // key = "20"
kvs[1], // key = "11"
kvs[0], // key = "10"
}
check(t, goodprefix, start, goodkvs, true, nil)
})
t.Run("no matching items", func(t *testing.T) {
goodprefix := []byte{}
start := []byte("0") // start is less than all keys.
check(t, goodprefix, start, []KeyValue{}, true, nil)
})
t.Run("early stop", func(t *testing.T) {
goodprefix := []byte{}
start := []byte("21")
goodkvs := []KeyValue{
kvs[3], // key = "21"
kvs[2], // key = "20"
kvs[1], // key = "11"
}
check(t, goodprefix, start, goodkvs, true, func(k, v []byte) bool {
return string(k) > "11"
})
})
})
})
t.Run("empty prefix, empty start", func(t *testing.T) {
goodprefix := []byte{}
start := []byte{}
goodkvs := make([]KeyValue, len(kvs))
copy(goodkvs, kvs)
t.Run("forwards", func(t *testing.T) {
t.Run("good", func(t *testing.T) {
check(t, goodprefix, start, goodkvs, false, nil)
})
t.Run("early stop", func(t *testing.T) {
goodkvs := []KeyValue{
kvs[0], // key = "10"
kvs[1], // key = "11"
kvs[2], // key = "20"
kvs[3], // key = "21"
}
check(t, goodprefix, start, goodkvs, false, func(k, v []byte) bool {
return string(k) < "21"
})
})
})
t.Run("backwards", func(t *testing.T) {
t.Run("good", func(t *testing.T) {
check(t, goodprefix, start, goodkvs, true, nil)
})
t.Run("early stop", func(t *testing.T) {
goodkvs := []KeyValue{
kvs[6], // key = "31"
kvs[5], // key = "30"
kvs[4], // key = "22"
kvs[3], // key = "21"
}
check(t, goodprefix, start, goodkvs, true, func(k, v []byte) bool {
return string(k) > "21"
})
})
})
})
}
func testStoreDeleteNonExistent(t *testing.T, s Store) {
key := []byte("sparse")
assert.NoError(t, s.Delete(key))
}
func testStorePutAndDelete(t *testing.T, s Store) {
key := []byte("foo")
value := []byte("bar")
require.NoError(t, s.Put(key, value))
err := s.Delete(key)
assert.Nil(t, err)
_, err = s.Get(key)
assert.NotNil(t, err)
assert.Equal(t, err, ErrKeyNotFound)
// Double delete.
err = s.Delete(key)
assert.Nil(t, err)
}
func testStorePutBatchWithDelete(t *testing.T, s Store) {
var (
toBeStored = map[string][]byte{
"foo": []byte("bar"),
"bar": []byte("baz"),
}
deletedInBatch = map[string][]byte{
"edc": []byte("rfv"),
"tgb": []byte("yhn"),
}
readdedToBatch = map[string][]byte{
"yhn": []byte("ujm"),
}
toBeDeleted = map[string][]byte{
"qaz": []byte("wsx"),
"qwe": []byte("123"),
}
toStay = map[string][]byte{
"key": []byte("val"),
"faa": []byte("bra"),
}
)
for k, v := range toBeDeleted {
require.NoError(t, s.Put([]byte(k), v))
}
for k, v := range toStay {
require.NoError(t, s.Put([]byte(k), v))
}
batch := s.Batch()
for k, v := range toBeStored {
batch.Put([]byte(k), v)
}
for k := range toBeDeleted {
batch.Delete([]byte(k))
}
for k, v := range readdedToBatch {
batch.Put([]byte(k), v)
}
for k, v := range deletedInBatch {
batch.Put([]byte(k), v)
}
for k := range deletedInBatch {
batch.Delete([]byte(k))
}
for k := range readdedToBatch {
batch.Delete([]byte(k))
}
for k, v := range readdedToBatch {
batch.Put([]byte(k), v)
}
require.NoError(t, s.PutBatch(batch))
toBe := []map[string][]byte{toStay, toBeStored, readdedToBatch}
notToBe := []map[string][]byte{deletedInBatch, toBeDeleted}
for _, kvs := range toBe {
for k, v := range kvs {
value, err := s.Get([]byte(k))
assert.Nil(t, err)
assert.Equal(t, value, v)
}
}
for _, kvs := range notToBe {
for k, v := range kvs {
_, err := s.Get([]byte(k))
assert.Equal(t, ErrKeyNotFound, err, "%s:%s", k, v)
}
}
} }
func testStoreSeekGC(t *testing.T, s Store) { func testStoreSeekGC(t *testing.T, s Store) {
kvs := []KeyValue{ kvs := pushSeekDataSet(t, s)
{[]byte("10"), []byte("bar")},
{[]byte("11"), []byte("bara")},
{[]byte("20"), []byte("barb")},
{[]byte("21"), []byte("barc")},
{[]byte("22"), []byte("bard")},
{[]byte("30"), []byte("bare")},
{[]byte("31"), []byte("barf")},
}
for _, v := range kvs {
require.NoError(t, s.Put(v.Key, v.Value))
}
err := s.SeekGC(SeekRange{Prefix: []byte("1")}, func(k, v []byte) bool { err := s.SeekGC(SeekRange{Prefix: []byte("1")}, func(k, v []byte) bool {
return true return true
}) })
@ -466,10 +236,8 @@ func TestAllDBs(t *testing.T) {
{"MemCached", newMemCachedStoreForTesting}, {"MemCached", newMemCachedStoreForTesting},
{"Memory", newMemoryStoreForTesting}, {"Memory", newMemoryStoreForTesting},
} }
var tests = []dbTestFunction{testStorePutAndGet, var tests = []dbTestFunction{testStoreGetNonExistent, testStoreSeek,
testStoreGetNonExistent, testStorePutBatch, testStoreSeek, testStoreSeekGC}
testStoreDeleteNonExistent, testStorePutAndDelete,
testStorePutBatchWithDelete, testStoreSeekGC}
for _, db := range DBs { for _, db := range DBs {
for _, test := range tests { for _, test := range tests {
s := db.create(t) s := db.create(t)