core: add cachedDao to cache accounts and contracts

Importing 100K blocks starting at 1.4M, before this patch:
real    6m0,356s
user    8m52,293s
sys     0m47,372s

After this patch:
real    4m17,748s
user    6m23,316s
sys     0m37,866s

Almost 30% better.

pkg/core/blockchain.go

@@ -344,7 +344,7 @@ func (bc *Blockchain) processHeader(h *Header, batch storage.Batch, headerList *
 // is happening here, quite allot as you can see :). If things are wired together
 // and all tests are in place, we can make a more optimized and cleaner implementation.
 func (bc *Blockchain) storeBlock(block *Block) error {
-	cache := newDao(bc.dao.store)
+	cache := newCachedDao(bc.dao.store)
 	if err := cache.StoreAsBlock(block, 0); err != nil {
 		return err
 	}
@@ -567,7 +567,7 @@ func (bc *Blockchain) storeBlock(block *Block) error {
 }
 
 // processOutputs processes transaction outputs.
-func processOutputs(tx *transaction.Transaction, dao *dao) error {
+func processOutputs(tx *transaction.Transaction, dao *cachedDao) error {
 	for index, output := range tx.Outputs {
 		account, err := dao.GetAccountStateOrNew(output.ScriptHash)
 		if err != nil {
@@ -588,7 +588,7 @@ func processOutputs(tx *transaction.Transaction, dao *dao) error {
 	return nil
 }
 
-func processTXWithValidatorsAdd(output *transaction.Output, account *state.Account, dao *dao) error {
+func processTXWithValidatorsAdd(output *transaction.Output, account *state.Account, dao *cachedDao) error {
 	if output.AssetID.Equals(governingTokenTX().Hash()) && len(account.Votes) > 0 {
 		for _, vote := range account.Votes {
 			validatorState, err := dao.GetValidatorStateOrNew(vote)
@@ -604,7 +604,7 @@ func processTXWithValidatorsAdd(output *transaction.Output, account *state.Accou
 	return nil
 }
 
-func processTXWithValidatorsSubtract(account *state.Account, dao *dao, toSubtract util.Fixed8) error {
+func processTXWithValidatorsSubtract(account *state.Account, dao *cachedDao, toSubtract util.Fixed8) error {
 	for _, vote := range account.Votes {
 		validator, err := dao.GetValidatorStateOrNew(vote)
 		if err != nil {
@@ -624,7 +624,7 @@ func processTXWithValidatorsSubtract(account *state.Account, dao *dao, toSubtrac
 	return nil
 }
 
-func processValidatorStateDescriptor(descriptor *transaction.StateDescriptor, dao *dao) error {
+func processValidatorStateDescriptor(descriptor *transaction.StateDescriptor, dao *cachedDao) error {
 	publicKey := &keys.PublicKey{}
 	err := publicKey.DecodeBytes(descriptor.Key)
 	if err != nil {
@@ -645,7 +645,7 @@ func processValidatorStateDescriptor(descriptor *transaction.StateDescriptor, da
 	return nil
 }
 
-func processAccountStateDescriptor(descriptor *transaction.StateDescriptor, dao *dao) error {
+func processAccountStateDescriptor(descriptor *transaction.StateDescriptor, dao *cachedDao) error {
 	hash, err := util.Uint160DecodeBytesBE(descriptor.Key)
 	if err != nil {
 		return err
@@ -1156,7 +1156,7 @@ func (bc *Blockchain) GetStandByValidators() (keys.PublicKeys, error) {
 // GetValidators returns validators.
 // Golang implementation of GetValidators method in C# (https://github.com/neo-project/neo/blob/c64748ecbac3baeb8045b16af0d518398a6ced24/neo/Persistence/Snapshot.cs#L182)
 func (bc *Blockchain) GetValidators(txes ...*transaction.Transaction) ([]*keys.PublicKey, error) {
-	cache := newDao(bc.dao.store)
+	cache := newCachedDao(bc.dao.store)
 	if len(txes) > 0 {
 		for _, tx := range txes {
 			// iterate through outputs
@@ -1252,7 +1252,7 @@ func (bc *Blockchain) GetValidators(txes ...*transaction.Transaction) ([]*keys.P
 	return result, nil
 }
 
-func processStateTX(dao *dao, tx *transaction.StateTX) error {
+func processStateTX(dao *cachedDao, tx *transaction.StateTX) error {
 	for _, desc := range tx.Descriptors {
 		switch desc.Type {
 		case transaction.Account:
@@ -1268,7 +1268,7 @@ func processStateTX(dao *dao, tx *transaction.StateTX) error {
 	return nil
 }
 
-func processEnrollmentTX(dao *dao, tx *transaction.EnrollmentTX) error {
+func processEnrollmentTX(dao *cachedDao, tx *transaction.EnrollmentTX) error {
 	validatorState, err := dao.GetValidatorStateOrNew(&tx.PublicKey)
 	if err != nil {
 		return err

pkg/core/cacheddao.go

@@ -0,0 +1,82 @@
+package core
+
+import (
+	"github.com/CityOfZion/neo-go/pkg/core/state"
+	"github.com/CityOfZion/neo-go/pkg/core/storage"
+	"github.com/CityOfZion/neo-go/pkg/util"
+)
+
+// cachedDao is a data access object that mimics dao, but has a write cache
+// for accounts and read cache for contracts. These are the most frequently used
+// objects in the storeBlock().
+type cachedDao struct {
+	dao
+	accounts  map[util.Uint160]*state.Account
+	contracts map[util.Uint160]*state.Contract
+}
+
+// newCachedDao returns new cachedDao wrapping around given backing store.
+func newCachedDao(backend storage.Store) *cachedDao {
+	accs := make(map[util.Uint160]*state.Account)
+	ctrs := make(map[util.Uint160]*state.Contract)
+	return &cachedDao{*newDao(backend), accs, ctrs}
+}
+
+// GetAccountStateOrNew retrieves Account from cache or underlying Store
+// or creates a new one if it doesn't exist.
+func (cd *cachedDao) GetAccountStateOrNew(hash util.Uint160) (*state.Account, error) {
+	if cd.accounts[hash] != nil {
+		return cd.accounts[hash], nil
+	}
+	return cd.dao.GetAccountStateOrNew(hash)
+}
+
+// GetAccountState retrieves Account from cache or underlying Store.
+func (cd *cachedDao) GetAccountState(hash util.Uint160) (*state.Account, error) {
+	if cd.accounts[hash] != nil {
+		return cd.accounts[hash], nil
+	}
+	return cd.dao.GetAccountState(hash)
+}
+
+// PutAccountState saves given Account in the cache.
+func (cd *cachedDao) PutAccountState(as *state.Account) error {
+	cd.accounts[as.ScriptHash] = as
+	return nil
+}
+
+// GetContractState returns contract state from cache or underlying Store.
+func (cd *cachedDao) GetContractState(hash util.Uint160) (*state.Contract, error) {
+	if cd.contracts[hash] != nil {
+		return cd.contracts[hash], nil
+	}
+	cs, err := cd.dao.GetContractState(hash)
+	if err == nil {
+		cd.contracts[hash] = cs
+	}
+	return cs, err
+}
+
+// PutContractState puts given contract state into the given store.
+func (cd *cachedDao) PutContractState(cs *state.Contract) error {
+	cd.contracts[cs.ScriptHash()] = cs
+	return cd.dao.PutContractState(cs)
+}
+
+// DeleteContractState deletes given contract state in cache and backing Store.
+func (cd *cachedDao) DeleteContractState(hash util.Uint160) error {
+	cd.contracts[hash] = nil
+	return cd.dao.DeleteContractState(hash)
+}
+
+// Persist flushes all the changes made into the (supposedly) persistent
+// underlying store.
+func (cd *cachedDao) Persist() (int, error) {
+	for sc := range cd.accounts {
+		err := cd.dao.PutAccountState(cd.accounts[sc])
+		if err != nil {
+			return 0, err
+		}
+	}
+	return cd.dao.Persist()
+}

pkg/core/cacheddao_test.go

@@ -0,0 +1,83 @@
+package core
+
+import (
+	"testing"
+
+	"github.com/CityOfZion/neo-go/pkg/core/state"
+	"github.com/CityOfZion/neo-go/pkg/core/storage"
+	"github.com/CityOfZion/neo-go/pkg/crypto/hash"
+	"github.com/CityOfZion/neo-go/pkg/internal/random"
+	"github.com/CityOfZion/neo-go/pkg/smartcontract"
+	"github.com/stretchr/testify/require"
+)
+
+func TestCachedDaoAccounts(t *testing.T) {
+	store := storage.NewMemoryStore()
+	// Persistent DAO to check for backing storage.
+	pdao := newDao(store)
+	// Cached DAO.
+	cdao := newCachedDao(store)
+
+	hash := random.Uint160()
+	_, err := cdao.GetAccountState(hash)
+	require.NotNil(t, err)
+
+	acc, err := cdao.GetAccountStateOrNew(hash)
+	require.Nil(t, err)
+	_, err = pdao.GetAccountState(hash)
+	require.NotNil(t, err)
+
+	acc.Version = 42
+	require.NoError(t, cdao.PutAccountState(acc))
+	_, err = pdao.GetAccountState(hash)
+	require.NotNil(t, err)
+
+	acc2, err := cdao.GetAccountState(hash)
+	require.Nil(t, err)
+	require.Equal(t, acc, acc2)
+
+	acc2, err = cdao.GetAccountStateOrNew(hash)
+	require.Nil(t, err)
+	require.Equal(t, acc, acc2)
+
+	_, err = cdao.Persist()
+	require.Nil(t, err)
+
+	acct, err := pdao.GetAccountState(hash)
+	require.Nil(t, err)
+	require.Equal(t, acc, acct)
+}
+
+func TestCachedDaoContracts(t *testing.T) {
+	store := storage.NewMemoryStore()
+	dao := newCachedDao(store)
+
+	script := []byte{0xde, 0xad, 0xbe, 0xef}
+	sh := hash.Hash160(script)
+	_, err := dao.GetContractState(sh)
+	require.NotNil(t, err)
+
+	cs := &state.Contract{}
+	cs.Name = "test"
+	cs.Script = script
+	cs.ParamList = []smartcontract.ParamType{1, 2}
+
+	require.NoError(t, dao.PutContractState(cs))
+	cs2, err := dao.GetContractState(sh)
+	require.Nil(t, err)
+	require.Equal(t, cs, cs2)
+
+	_, err = dao.Persist()
+	require.Nil(t, err)
+	dao2 := newCachedDao(store)
+	cs2, err = dao2.GetContractState(sh)
+	require.Nil(t, err)
+	require.Equal(t, cs, cs2)
+
+	require.NoError(t, dao.DeleteContractState(sh))
+	cs2, err = dao2.GetContractState(sh)
+	require.Nil(t, err)
+	require.Equal(t, cs, cs2)
+	_, err = dao.GetContractState(sh)
+	require.NotNil(t, err)
+}

pkg/core/interops.go

@@ -19,12 +19,12 @@ type interopContext struct {
 	trigger       byte
 	block         *Block
 	tx            *transaction.Transaction
-	dao           *dao
+	dao           *cachedDao
 	notifications []state.NotificationEvent
 }
 
 func newInteropContext(trigger byte, bc Blockchainer, s storage.Store, block *Block, tx *transaction.Transaction) *interopContext {
-	dao := newDao(s)
+	dao := newCachedDao(s)
 	nes := make([]state.NotificationEvent, 0)
 	return &interopContext{bc, trigger, block, tx, dao, nes}
 }