Merge pull request #1396 from nspcc-dev/tps-part-2-clean

Improve TPS, part 2
This commit is contained in:
Roman Khimov 2020-09-09 23:23:32 +03:00 committed by GitHub
commit 63bb36ca12
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
30 changed files with 271 additions and 144 deletions

3
go.mod
View file

@ -8,8 +8,9 @@ require (
github.com/frankban/quicktest v1.10.0 // indirect
github.com/go-redis/redis v6.10.2+incompatible
github.com/gorilla/websocket v1.4.2
github.com/hashicorp/golang-lru v0.5.4
github.com/mr-tron/base58 v1.1.2
github.com/nspcc-dev/dbft v0.0.0-20200711144034-c526ccc6f570
github.com/nspcc-dev/dbft v0.0.0-20200904131615-4443b3066b8b
github.com/nspcc-dev/rfc6979 v0.2.0
github.com/pierrec/lz4 v2.5.2+incompatible
github.com/prometheus/client_golang v1.2.1

7
go.sum
View file

@ -2,6 +2,7 @@ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03
github.com/CityOfZion/neo-go v0.62.1-pre.0.20191114145240-e740fbe708f8/go.mod h1:MJCkWUBhi9pn/CrYO1Q3P687y2KeahrOPS9BD9LDGb0=
github.com/CityOfZion/neo-go v0.70.1-pre.0.20191209120015-fccb0085941e/go.mod h1:0enZl0az8xA6PVkwzEOwPWVJGqlt/GO4hA4kmQ5Xzig=
github.com/CityOfZion/neo-go v0.70.1-pre.0.20191212173117-32ac01130d4c/go.mod h1:JtlHfeqLywZLswKIKFnAp+yzezY4Dji9qlfQKB2OD/I=
github.com/CityOfZion/neo-go v0.71.1-pre.0.20200129171427-f773ec69fb84 h1:gcTXk9aO+PhHudJNPFJ9H4RmKjdzz40Tvv2NE1BwRZ0=
github.com/CityOfZion/neo-go v0.71.1-pre.0.20200129171427-f773ec69fb84/go.mod h1:FLI526IrRWHmcsO+mHsCbj64pJZhwQFTLJZu+A4PGOA=
github.com/DataDog/zstd v1.4.1 h1:3oxKN3wbHibqx897utPC2LTQU4J+IHWWJO+glkAkpFM=
github.com/DataDog/zstd v1.4.1/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo=
@ -110,6 +111,8 @@ github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/
github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc=
github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
github.com/hashicorp/golang-lru v0.5.4 h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+lJfyTc=
github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4=
github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI=
github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
@ -162,8 +165,8 @@ github.com/nspcc-dev/dbft v0.0.0-20200117124306-478e5cfbf03a h1:ajvxgEe9qY4vvoSm
github.com/nspcc-dev/dbft v0.0.0-20200117124306-478e5cfbf03a/go.mod h1:/YFK+XOxxg0Bfm6P92lY5eDSLYfp06XOdL8KAVgXjVk=
github.com/nspcc-dev/dbft v0.0.0-20200219114139-199d286ed6c1 h1:yEx9WznS+rjE0jl0dLujCxuZSIb+UTjF+005TJu/nNI=
github.com/nspcc-dev/dbft v0.0.0-20200219114139-199d286ed6c1/go.mod h1:O0qtn62prQSqizzoagHmuuKoz8QMkU3SzBoKdEvm3aQ=
github.com/nspcc-dev/dbft v0.0.0-20200711144034-c526ccc6f570 h1:EHBwlOyd2m06C3dnxhpPokpYqlNg7u5ZX/uPBhjYuZ4=
github.com/nspcc-dev/dbft v0.0.0-20200711144034-c526ccc6f570/go.mod h1:1FYQXSbb6/9HQIkoF8XO7W/S8N7AZRkBsgwbcXRvk0E=
github.com/nspcc-dev/dbft v0.0.0-20200904131615-4443b3066b8b h1:UgjsL0bhHWF/g7iK673brucYjwpB4MM0Sgg2iVaOYGc=
github.com/nspcc-dev/dbft v0.0.0-20200904131615-4443b3066b8b/go.mod h1:1FYQXSbb6/9HQIkoF8XO7W/S8N7AZRkBsgwbcXRvk0E=
github.com/nspcc-dev/neo-go v0.73.1-pre.0.20200303142215-f5a1b928ce09/go.mod h1:pPYwPZ2ks+uMnlRLUyXOpLieaDQSEaf4NM3zHVbRjmg=
github.com/nspcc-dev/neofs-crypto v0.2.0 h1:ftN+59WqxSWz/RCgXYOfhmltOOqU+udsNQSvN6wkFck=
github.com/nspcc-dev/neofs-crypto v0.2.0/go.mod h1:F/96fUzPM3wR+UGsPi3faVNmFlA9KAEAUQR7dMxZmNA=

View file

@ -155,6 +155,7 @@ func NewService(cfg Config) (Service, error) {
dbft.WithNewRecoveryRequest(func() payload.RecoveryRequest { return new(recoveryRequest) }),
dbft.WithNewRecoveryMessage(func() payload.RecoveryMessage { return new(recoveryMessage) }),
dbft.WithVerifyPrepareRequest(srv.verifyRequest),
dbft.WithVerifyPrepareResponse(func(_ payload.ConsensusPayload) error { return nil }),
)
if srv.dbft == nil {

View file

@ -35,6 +35,10 @@ type (
height uint32
Witness transaction.Witness
hash util.Uint256
signedHash util.Uint256
signedpart []byte
}
)
@ -106,11 +110,13 @@ func (p Payload) GetRecoveryMessage() payload.RecoveryMessage {
}
// MarshalUnsigned implements payload.ConsensusPayload interface.
func (p Payload) MarshalUnsigned() []byte {
w := io.NewBufBinWriter()
p.encodeHashData(w.BinWriter)
return w.Bytes()
func (p *Payload) MarshalUnsigned() []byte {
if p.signedpart == nil {
w := io.NewBufBinWriter()
p.encodeHashData(w.BinWriter)
p.signedpart = w.Bytes()
}
return p.signedpart
}
// UnmarshalUnsigned implements payload.ConsensusPayload interface.
@ -179,7 +185,10 @@ func (p *Payload) EncodeBinaryUnsigned(w *io.BinWriter) {
// EncodeBinary implements io.Serializable interface.
func (p *Payload) EncodeBinary(w *io.BinWriter) {
p.EncodeBinaryUnsigned(w)
if p.signedpart == nil {
_ = p.MarshalUnsigned()
}
w.WriteBytes(p.signedpart[4:])
w.WriteB(1)
p.Witness.EncodeBinary(w)
@ -193,10 +202,7 @@ func (p *Payload) encodeHashData(w *io.BinWriter) {
// Sign signs payload using the private key.
// It also sets corresponding verification and invocation scripts.
func (p *Payload) Sign(key *privateKey) error {
sig, err := key.Sign(p.GetSignedPart())
if err != nil {
return err
}
sig := key.SignHash(p.GetSignedHash())
buf := io.NewBufBinWriter()
emit.Bytes(buf.BinWriter, sig)
@ -224,15 +230,41 @@ func (p *Payload) DecodeBinaryUnsigned(r *io.BinReader) {
}
}
// GetSignedHash returns a hash of the payload used to verify it.
func (p *Payload) GetSignedHash() util.Uint256 {
if p.signedHash.Equals(util.Uint256{}) {
if p.createHash() != nil {
panic("failed to compute hash!")
}
}
return p.signedHash
}
// Hash implements payload.ConsensusPayload interface.
func (p *Payload) Hash() util.Uint256 {
w := io.NewBufBinWriter()
p.encodeHashData(w.BinWriter)
if w.Err != nil {
panic("failed to hash payload")
if p.hash.Equals(util.Uint256{}) {
if p.createHash() != nil {
panic("failed to compute hash!")
}
}
return p.hash
}
return hash.DoubleSha256(w.Bytes())
// createHash creates hashes of the payload.
func (p *Payload) createHash() error {
b := p.GetSignedPart()
if b == nil {
return errors.New("failed to serialize hashable data")
}
p.updateHashes(b)
return nil
}
// updateHashes updates Payload's hashes based on the given buffer which should
// be a signable data slice.
func (p *Payload) updateHashes(b []byte) {
p.signedHash = hash.Sha256(b)
p.hash = hash.Sha256(p.signedHash.BytesBE())
}
// DecodeBinary implements io.Serializable interface.

View file

@ -104,12 +104,14 @@ func TestConsensusPayload_Serializable(t *testing.T) {
require.Nil(t, actual.message)
// message should now be decoded from actual.data byte array
assert.NoError(t, actual.decodeData())
assert.NotNil(t, actual.MarshalUnsigned())
require.Equal(t, p, actual)
data = p.MarshalUnsigned()
pu := NewPayload(netmode.Magic(rand.Uint32()))
require.NoError(t, pu.UnmarshalUnsigned(data))
assert.NoError(t, pu.decodeData())
_ = pu.MarshalUnsigned()
p.Witness = transaction.Witness{}
require.Equal(t, p, pu)
@ -153,6 +155,7 @@ func TestConsensusPayload_DecodeBinaryInvalid(t *testing.T) {
p := new(Payload)
require.NoError(t, testserdes.DecodeBinary(buf, p))
// decode `data` into `message`
_ = p.Hash()
assert.NoError(t, p.decodeData())
require.Equal(t, expected, p)

View file

@ -57,6 +57,8 @@ func TestRecoveryMessage_Setters(t *testing.T) {
ps := r.GetPrepareResponses(p, pubs)
require.Len(t, ps, 1)
// Update hashes and serialized data.
_ = ps[0].Hash()
require.Equal(t, p2, ps[0])
ps0 := ps[0].(*Payload)
require.True(t, srv.validatePayload(ps0))
@ -91,6 +93,8 @@ func TestRecoveryMessage_Setters(t *testing.T) {
ps := r.GetChangeViews(p, pubs)
require.Len(t, ps, 1)
// update hashes and serialized data.
_ = ps[0].Hash()
require.Equal(t, p3, ps[0])
ps0 := ps[0].(*Payload)
@ -109,6 +113,8 @@ func TestRecoveryMessage_Setters(t *testing.T) {
ps := r.GetCommits(p, pubs)
require.Len(t, ps, 1)
// update hashes and serialized data.
_ = ps[0].Hash()
require.Equal(t, p4, ps[0])
ps0 := ps[0].(*Payload)

View file

@ -78,8 +78,8 @@ func (b *Base) Hash() util.Uint256 {
return b.hash
}
// VerificationHash returns the hash of the block used to verify it.
func (b *Base) VerificationHash() util.Uint256 {
// GetSignedHash returns a hash of the block used to verify it.
func (b *Base) GetSignedHash() util.Uint256 {
if b.verificationHash.Equals(util.Uint256{}) {
b.createHash()
}

View file

@ -577,14 +577,17 @@ func (bc *Blockchain) GetStateRoot(height uint32) (*state.MPTRootState, error) {
// This is the only way to change Blockchain state.
func (bc *Blockchain) storeBlock(block *block.Block) error {
cache := dao.NewCached(bc.dao)
writeBuf := io.NewBufBinWriter()
appExecResults := make([]*state.AppExecResult, 0, 1+len(block.Transactions))
if err := cache.StoreAsBlock(block); err != nil {
if err := cache.StoreAsBlock(block, writeBuf); err != nil {
return err
}
writeBuf.Reset()
if err := cache.StoreAsCurrentBlock(block); err != nil {
if err := cache.StoreAsCurrentBlock(block, writeBuf); err != nil {
return err
}
writeBuf.Reset()
if block.Index > 0 {
systemInterop := bc.newInteropContext(trigger.System, cache, block, nil)
@ -608,17 +611,19 @@ func (bc *Blockchain) storeBlock(block *block.Block) error {
Events: systemInterop.Notifications,
}
appExecResults = append(appExecResults, aer)
err := cache.PutAppExecResult(aer)
err := cache.PutAppExecResult(aer, writeBuf)
if err != nil {
return fmt.Errorf("failed to store onPersist exec result: %w", err)
}
writeBuf.Reset()
}
var txHashes = make([]util.Uint256, len(block.Transactions))
for i, tx := range block.Transactions {
if err := cache.StoreAsTransaction(tx, block.Index); err != nil {
if err := cache.StoreAsTransaction(tx, block.Index, writeBuf); err != nil {
return err
}
writeBuf.Reset()
systemInterop := bc.newInteropContext(trigger.Application, cache, block, tx)
v := systemInterop.SpawnVM()
@ -650,10 +655,11 @@ func (bc *Blockchain) storeBlock(block *block.Block) error {
Events: systemInterop.Notifications,
}
appExecResults = append(appExecResults, aer)
err = cache.PutAppExecResult(aer)
err = cache.PutAppExecResult(aer, writeBuf)
if err != nil {
return fmt.Errorf("failed to store tx exec result: %w", err)
}
writeBuf.Reset()
txHashes[i] = tx.Hash()
}
sort.Slice(txHashes, func(i, j int) bool {

View file

@ -22,7 +22,6 @@ type Blockchainer interface {
AddHeaders(...*block.Header) error
AddBlock(*block.Block) error
AddStateRoot(r *state.MPTRoot) error
BlockHeight() uint32
CalculateClaimable(value *big.Int, startHeight, endHeight uint32) *big.Int
Close()
HeaderHeight() uint32

View file

@ -44,16 +44,16 @@ type DAO interface {
GetWrapped() DAO
HasTransaction(hash util.Uint256) bool
Persist() (int, error)
PutAppExecResult(aer *state.AppExecResult) error
PutAppExecResult(aer *state.AppExecResult, buf *io.BufBinWriter) error
PutContractState(cs *state.Contract) error
PutCurrentHeader(hashAndIndex []byte) error
PutNEP5Balances(acc util.Uint160, bs *state.NEP5Balances) error
PutNEP5TransferLog(acc util.Uint160, index uint32, lg *state.NEP5TransferLog) error
PutStorageItem(id int32, key []byte, si *state.StorageItem) error
PutVersion(v string) error
StoreAsBlock(block *block.Block) error
StoreAsCurrentBlock(block *block.Block) error
StoreAsTransaction(tx *transaction.Transaction, index uint32) error
StoreAsBlock(block *block.Block, buf *io.BufBinWriter) error
StoreAsCurrentBlock(block *block.Block, buf *io.BufBinWriter) error
StoreAsTransaction(tx *transaction.Transaction, index uint32, buf *io.BufBinWriter) error
putNEP5Balances(acc util.Uint160, bs *state.NEP5Balances, buf *io.BufBinWriter) error
}
@ -270,10 +270,13 @@ func (dao *Simple) GetAppExecResult(hash util.Uint256) (*state.AppExecResult, er
}
// PutAppExecResult puts given application execution result into the
// given store.
func (dao *Simple) PutAppExecResult(aer *state.AppExecResult) error {
// given store. It can reuse given buffer for the purpose of value serialization.
func (dao *Simple) PutAppExecResult(aer *state.AppExecResult, buf *io.BufBinWriter) error {
key := storage.AppendPrefix(storage.STNotification, aer.TxHash.BytesBE())
return dao.Put(aer, key)
if buf == nil {
return dao.Put(aer, key)
}
return dao.putWithBuffer(aer, key, buf)
}
// -- end notification event.
@ -560,12 +563,15 @@ func (dao *Simple) HasTransaction(hash util.Uint256) bool {
return false
}
// StoreAsBlock stores the given block as DataBlock.
func (dao *Simple) StoreAsBlock(block *block.Block) error {
// StoreAsBlock stores given block as DataBlock. It can reuse given buffer for
// the purpose of value serialization.
func (dao *Simple) StoreAsBlock(block *block.Block, buf *io.BufBinWriter) error {
var (
key = storage.AppendPrefix(storage.DataBlock, block.Hash().BytesLE())
buf = io.NewBufBinWriter()
)
if buf == nil {
buf = io.NewBufBinWriter()
}
b, err := block.Trim()
if err != nil {
return err
@ -577,19 +583,26 @@ func (dao *Simple) StoreAsBlock(block *block.Block) error {
return dao.Store.Put(key, buf.Bytes())
}
// StoreAsCurrentBlock stores the given block witch prefix SYSCurrentBlock.
func (dao *Simple) StoreAsCurrentBlock(block *block.Block) error {
buf := io.NewBufBinWriter()
// StoreAsCurrentBlock stores a hash of the given block with prefix
// SYSCurrentBlock. It can reuse given buffer for the purpose of value
// serialization.
func (dao *Simple) StoreAsCurrentBlock(block *block.Block, buf *io.BufBinWriter) error {
if buf == nil {
buf = io.NewBufBinWriter()
}
h := block.Hash()
h.EncodeBinary(buf.BinWriter)
buf.WriteU32LE(block.Index)
return dao.Store.Put(storage.SYSCurrentBlock.Bytes(), buf.Bytes())
}
// StoreAsTransaction stores the given TX as DataTransaction.
func (dao *Simple) StoreAsTransaction(tx *transaction.Transaction, index uint32) error {
// StoreAsTransaction stores given TX as DataTransaction. It can reuse given
// buffer for the purpose of value serialization.
func (dao *Simple) StoreAsTransaction(tx *transaction.Transaction, index uint32, buf *io.BufBinWriter) error {
key := storage.AppendPrefix(storage.DataTransaction, tx.Hash().BytesLE())
buf := io.NewBufBinWriter()
if buf == nil {
buf = io.NewBufBinWriter()
}
buf.WriteU32LE(index)
tx.EncodeBinary(buf.BinWriter)
if buf.Err != nil {

View file

@ -86,7 +86,7 @@ func TestPutGetAppExecResult(t *testing.T) {
Events: []state.NotificationEvent{},
Stack: []stackitem.Item{},
}
err := dao.PutAppExecResult(appExecResult)
err := dao.PutAppExecResult(appExecResult, nil)
require.NoError(t, err)
gotAppExecResult, err := dao.GetAppExecResult(hash)
require.NoError(t, err)
@ -136,7 +136,7 @@ func TestPutGetBlock(t *testing.T) {
},
}
hash := b.Hash()
err := dao.StoreAsBlock(b)
err := dao.StoreAsBlock(b, nil)
require.NoError(t, err)
gotBlock, err := dao.GetBlock(hash)
require.NoError(t, err)
@ -176,7 +176,7 @@ func TestGetCurrentHeaderHeight_Store(t *testing.T) {
},
},
}
err := dao.StoreAsCurrentBlock(b)
err := dao.StoreAsCurrentBlock(b, nil)
require.NoError(t, err)
height, err := dao.GetCurrentBlockHeight()
require.NoError(t, err)
@ -187,7 +187,7 @@ func TestStoreAsTransaction(t *testing.T) {
dao := NewSimple(storage.NewMemoryStore(), netmode.UnitTestNet)
tx := transaction.New(netmode.UnitTestNet, []byte{byte(opcode.PUSH1)}, 1)
hash := tx.Hash()
err := dao.StoreAsTransaction(tx, 0)
err := dao.StoreAsTransaction(tx, 0, nil)
require.NoError(t, err)
hasTransaction := dao.HasTransaction(hash)
require.True(t, hasTransaction)

View file

@ -10,6 +10,7 @@ import (
"github.com/nspcc-dev/neo-go/pkg/crypto"
"github.com/nspcc-dev/neo-go/pkg/crypto/hash"
"github.com/nspcc-dev/neo-go/pkg/crypto/keys"
"github.com/nspcc-dev/neo-go/pkg/util"
"github.com/nspcc-dev/neo-go/pkg/vm"
"github.com/nspcc-dev/neo-go/pkg/vm/stackitem"
)
@ -30,15 +31,17 @@ func ECDSASecp256k1Verify(ic *interop.Context) error {
// ecdsaVerify is internal representation of ECDSASecp256k1Verify and
// ECDSASecp256r1Verify.
func ecdsaVerify(ic *interop.Context, curve elliptic.Curve) error {
msg := getMessage(ic, ic.VM.Estack().Pop().Item())
hashToCheck := hash.Sha256(msg).BytesBE()
hashToCheck, err := getMessageHash(ic, ic.VM.Estack().Pop().Item())
if err != nil {
return err
}
keyb := ic.VM.Estack().Pop().Bytes()
signature := ic.VM.Estack().Pop().Bytes()
pkey, err := keys.NewPublicKeyFromBytes(keyb, curve)
if err != nil {
return err
}
res := pkey.Verify(signature, hashToCheck)
res := pkey.Verify(signature, hashToCheck.BytesBE())
ic.VM.Estack().PushVal(res)
return nil
}
@ -58,8 +61,10 @@ func ECDSASecp256k1CheckMultisig(ic *interop.Context) error {
// ecdsaCheckMultisig is internal representation of ECDSASecp256r1CheckMultisig and
// ECDSASecp256k1CheckMultisig
func ecdsaCheckMultisig(ic *interop.Context, curve elliptic.Curve) error {
msg := getMessage(ic, ic.VM.Estack().Pop().Item())
hashToCheck := hash.Sha256(msg).BytesBE()
hashToCheck, err := getMessageHash(ic, ic.VM.Estack().Pop().Item())
if err != nil {
return err
}
pkeys, err := ic.VM.Estack().PopSigElements()
if err != nil {
return fmt.Errorf("wrong parameters: %w", err)
@ -76,23 +81,23 @@ func ecdsaCheckMultisig(ic *interop.Context, curve elliptic.Curve) error {
if len(pkeys) < len(sigs) {
return errors.New("more signatures than there are keys")
}
sigok := vm.CheckMultisigPar(ic.VM, curve, hashToCheck, pkeys, sigs)
sigok := vm.CheckMultisigPar(ic.VM, curve, hashToCheck.BytesBE(), pkeys, sigs)
ic.VM.Estack().PushVal(sigok)
return nil
}
func getMessage(ic *interop.Context, item stackitem.Item) []byte {
func getMessageHash(ic *interop.Context, item stackitem.Item) (util.Uint256, error) {
var msg []byte
switch val := item.(type) {
case *stackitem.Interop:
msg = val.Value().(crypto.Verifiable).GetSignedPart()
return val.Value().(crypto.Verifiable).GetSignedHash(), nil
case stackitem.Null:
msg = ic.Container.GetSignedPart()
return ic.Container.GetSignedHash(), nil
default:
var err error
if msg, err = val.TryBytes(); err != nil {
return nil
return util.Uint256{}, err
}
}
return msg
return hash.Sha256(msg), nil
}

View file

@ -2,20 +2,37 @@ package crypto
import (
"github.com/nspcc-dev/neo-go/pkg/core/interop"
"github.com/nspcc-dev/neo-go/pkg/crypto"
"github.com/nspcc-dev/neo-go/pkg/crypto/hash"
"github.com/nspcc-dev/neo-go/pkg/vm/stackitem"
)
// Sha256 returns sha256 hash of the data.
func Sha256(ic *interop.Context) error {
msg := getMessage(ic, ic.VM.Estack().Pop().Item())
h := hash.Sha256(msg).BytesBE()
ic.VM.Estack().PushVal(h)
h, err := getMessageHash(ic, ic.VM.Estack().Pop().Item())
if err != nil {
return err
}
ic.VM.Estack().PushVal(h.BytesBE())
return nil
}
// RipeMD160 returns RipeMD160 hash of the data.
func RipeMD160(ic *interop.Context) error {
msg := getMessage(ic, ic.VM.Estack().Pop().Item())
var msg []byte
item := ic.VM.Estack().Pop().Item()
switch val := item.(type) {
case *stackitem.Interop:
msg = val.Value().(crypto.Verifiable).GetSignedPart()
case stackitem.Null:
msg = ic.Container.GetSignedPart()
default:
var err error
if msg, err = val.TryBytes(); err != nil {
return err
}
}
h := hash.RipeMD160(msg).BytesBE()
ic.VM.Estack().PushVal(h)
return nil

View file

@ -216,7 +216,7 @@ func TestECDSAVerify(t *testing.T) {
t.Run("invalid message", func(t *testing.T) {
sign := priv.Sign(msg)
runCase(t, false, false, sign, priv.PublicKey().Bytes(),
runCase(t, true, false, sign, priv.PublicKey().Bytes(),
stackitem.NewArray([]stackitem.Item{stackitem.NewByteArray(msg)}))
})
}

View file

@ -27,7 +27,7 @@ func TestBCGetTransaction(t *testing.T) {
defer chain.Close()
t.Run("success", func(t *testing.T) {
require.NoError(t, context.DAO.StoreAsTransaction(tx, 0))
require.NoError(t, context.DAO.StoreAsTransaction(tx, 0, nil))
v.Estack().PushVal(tx.Hash().BytesBE())
err := bcGetTransaction(context)
require.NoError(t, err)
@ -47,7 +47,7 @@ func TestBCGetTransaction(t *testing.T) {
})
t.Run("isn't traceable", func(t *testing.T) {
require.NoError(t, context.DAO.StoreAsTransaction(tx, 1))
require.NoError(t, context.DAO.StoreAsTransaction(tx, 1, nil))
v.Estack().PushVal(tx.Hash().BytesBE())
err := bcGetTransaction(context)
require.NoError(t, err)
@ -57,7 +57,7 @@ func TestBCGetTransaction(t *testing.T) {
})
t.Run("bad hash", func(t *testing.T) {
require.NoError(t, context.DAO.StoreAsTransaction(tx, 1))
require.NoError(t, context.DAO.StoreAsTransaction(tx, 1, nil))
v.Estack().PushVal(tx.Hash().BytesLE())
err := bcGetTransaction(context)
require.NoError(t, err)
@ -71,7 +71,7 @@ func TestBCGetTransactionFromBlock(t *testing.T) {
v, block, context, chain := createVMAndBlock(t)
defer chain.Close()
require.NoError(t, chain.AddBlock(chain.newBlock()))
require.NoError(t, context.DAO.StoreAsBlock(block))
require.NoError(t, context.DAO.StoreAsBlock(block, nil))
t.Run("success", func(t *testing.T) {
v.Estack().PushVal(0)
@ -94,7 +94,7 @@ func TestBCGetTransactionFromBlock(t *testing.T) {
t.Run("isn't traceable", func(t *testing.T) {
block.Index = 2
require.NoError(t, context.DAO.StoreAsBlock(block))
require.NoError(t, context.DAO.StoreAsBlock(block, nil))
v.Estack().PushVal(0)
v.Estack().PushVal(block.Hash().BytesBE())
err := bcGetTransactionFromBlock(context)
@ -106,7 +106,7 @@ func TestBCGetTransactionFromBlock(t *testing.T) {
t.Run("bad block hash", func(t *testing.T) {
block.Index = 1
require.NoError(t, context.DAO.StoreAsBlock(block))
require.NoError(t, context.DAO.StoreAsBlock(block, nil))
v.Estack().PushVal(0)
v.Estack().PushVal(block.Hash().BytesLE())
err := bcGetTransactionFromBlock(context)
@ -117,7 +117,7 @@ func TestBCGetTransactionFromBlock(t *testing.T) {
})
t.Run("bad transaction index", func(t *testing.T) {
require.NoError(t, context.DAO.StoreAsBlock(block))
require.NoError(t, context.DAO.StoreAsBlock(block, nil))
v.Estack().PushVal(1)
v.Estack().PushVal(block.Hash().BytesBE())
err := bcGetTransactionFromBlock(context)

View file

@ -10,4 +10,5 @@ import (
type Feer interface {
FeePerByte() int64
GetUtilityTokenBalance(util.Uint160) *big.Int
BlockHeight() uint32
}

View file

@ -5,7 +5,6 @@ import (
"math/big"
"sort"
"sync"
"time"
"github.com/nspcc-dev/neo-go/pkg/core/transaction"
"github.com/nspcc-dev/neo-go/pkg/util"
@ -30,12 +29,12 @@ var (
// item represents a transaction in the the Memory pool.
type item struct {
txn *transaction.Transaction
timeStamp time.Time
txn *transaction.Transaction
blockStamp uint32
}
// items is a slice of item.
type items []*item
type items []item
// utilityBalanceAndFees stores sender's balance and overall fees of
// sender's transactions which are currently in mempool
@ -47,7 +46,7 @@ type utilityBalanceAndFees struct {
// Pool stores the unconfirms transactions.
type Pool struct {
lock sync.RWMutex
verifiedMap map[util.Uint256]*item
verifiedMap map[util.Uint256]*transaction.Transaction
verifiedTxes items
fees map[util.Uint160]utilityBalanceAndFees
@ -63,11 +62,7 @@ func (p items) Less(i, j int) bool { return p[i].CompareTo(p[j]) < 0 }
// difference < 0 implies p < otherP.
// difference = 0 implies p = otherP.
// difference > 0 implies p > otherP.
func (p *item) CompareTo(otherP *item) int {
if otherP == nil {
return 1
}
func (p item) CompareTo(otherP item) int {
pHigh := p.txn.HasAttribute(transaction.HighPriority)
otherHigh := otherP.txn.HasAttribute(transaction.HighPriority)
if pHigh && !otherHigh {
@ -81,12 +76,7 @@ func (p *item) CompareTo(otherP *item) int {
return ret
}
if ret := int(p.txn.NetworkFee - otherP.txn.NetworkFee); ret != 0 {
return ret
}
// Transaction hash sorted descending.
return otherP.txn.Hash().CompareTo(p.txn.Hash())
return int(p.txn.NetworkFee - otherP.txn.NetworkFee)
}
// Count returns the total number of uncofirm transactions.
@ -151,9 +141,9 @@ func checkBalance(tx *transaction.Transaction, balance utilityBalanceAndFees) er
// Add tries to add given transaction to the Pool.
func (mp *Pool) Add(t *transaction.Transaction, fee Feer) error {
var pItem = &item{
txn: t,
timeStamp: time.Now().UTC(),
var pItem = item{
txn: t,
blockStamp: fee.BlockHeight(),
}
mp.lock.Lock()
if mp.containsKey(t.Hash()) {
@ -166,7 +156,7 @@ func (mp *Pool) Add(t *transaction.Transaction, fee Feer) error {
return err
}
mp.verifiedMap[t.Hash()] = pItem
mp.verifiedMap[t.Hash()] = t
// Insert into sorted array (from max to min, that could also be done
// using sort.Sort(sort.Reverse()), but it incurs more overhead. Notice
// also that we're searching for position that is strictly more
@ -207,7 +197,7 @@ func (mp *Pool) Add(t *transaction.Transaction, fee Feer) error {
// nothing if it doesn't).
func (mp *Pool) Remove(hash util.Uint256) {
mp.lock.Lock()
if it, ok := mp.verifiedMap[hash]; ok {
if tx, ok := mp.verifiedMap[hash]; ok {
var num int
delete(mp.verifiedMap, hash)
for num = range mp.verifiedTxes {
@ -220,9 +210,9 @@ func (mp *Pool) Remove(hash util.Uint256) {
} else if num == len(mp.verifiedTxes)-1 {
mp.verifiedTxes = mp.verifiedTxes[:num]
}
senderFee := mp.fees[it.txn.Sender()]
senderFee.feeSum.Sub(senderFee.feeSum, big.NewInt(it.txn.SystemFee+it.txn.NetworkFee))
mp.fees[it.txn.Sender()] = senderFee
senderFee := mp.fees[tx.Sender()]
senderFee.feeSum.Sub(senderFee.feeSum, big.NewInt(tx.SystemFee+tx.NetworkFee))
mp.fees[tx.Sender()] = senderFee
}
updateMempoolMetrics(len(mp.verifiedTxes))
mp.lock.Unlock()
@ -271,8 +261,8 @@ func (mp *Pool) checkPolicy(tx *transaction.Transaction, policyChanged bool) boo
// New returns a new Pool struct.
func New(capacity int) *Pool {
return &Pool{
verifiedMap: make(map[util.Uint256]*item),
verifiedTxes: make([]*item, 0, capacity),
verifiedMap: make(map[util.Uint256]*transaction.Transaction),
verifiedTxes: make([]item, 0, capacity),
capacity: capacity,
fees: make(map[util.Uint160]utilityBalanceAndFees),
}
@ -282,8 +272,8 @@ func New(capacity int) *Pool {
func (mp *Pool) TryGetValue(hash util.Uint256) (*transaction.Transaction, bool) {
mp.lock.RLock()
defer mp.lock.RUnlock()
if pItem, ok := mp.verifiedMap[hash]; ok {
return pItem.txn, ok
if tx, ok := mp.verifiedMap[hash]; ok {
return tx, ok
}
return nil, false

View file

@ -23,6 +23,10 @@ func (fs *FeerStub) FeePerByte() int64 {
return fs.feePerByte
}
func (fs *FeerStub) BlockHeight() uint32 {
return 0
}
func (fs *FeerStub) GetUtilityTokenBalance(uint160 util.Uint160) *big.Int {
return balance
}
@ -262,23 +266,23 @@ func TestMempoolItemsOrder(t *testing.T) {
tx1.NetworkFee = new(big.Int).Div(balance, big.NewInt(8)).Int64()
tx1.Signers = []transaction.Signer{{Account: sender0}}
tx1.Attributes = []transaction.Attribute{{Type: transaction.HighPriority}}
item1 := &item{txn: tx1}
item1 := item{txn: tx1}
tx2 := transaction.New(netmode.UnitTestNet, []byte{byte(opcode.PUSH1)}, 0)
tx2.NetworkFee = new(big.Int).Div(balance, big.NewInt(16)).Int64()
tx2.Signers = []transaction.Signer{{Account: sender0}}
tx2.Attributes = []transaction.Attribute{{Type: transaction.HighPriority}}
item2 := &item{txn: tx2}
item2 := item{txn: tx2}
tx3 := transaction.New(netmode.UnitTestNet, []byte{byte(opcode.PUSH1)}, 0)
tx3.NetworkFee = new(big.Int).Div(balance, big.NewInt(2)).Int64()
tx3.Signers = []transaction.Signer{{Account: sender0}}
item3 := &item{txn: tx3}
item3 := item{txn: tx3}
tx4 := transaction.New(netmode.UnitTestNet, []byte{byte(opcode.PUSH1)}, 0)
tx4.NetworkFee = new(big.Int).Div(balance, big.NewInt(4)).Int64()
tx4.Signers = []transaction.Signer{{Account: sender0}}
item4 := &item{txn: tx4}
item4 := item{txn: tx4}
require.True(t, item1.CompareTo(item2) > 0)
require.True(t, item2.CompareTo(item1) < 0)

View file

@ -59,6 +59,13 @@ func (s *MPTRootBase) GetSignedPart() []byte {
return buf.Bytes()
}
// GetSignedHash returns hash of MPTRootBase which needs to be signed.
func (s *MPTRootBase) GetSignedHash() util.Uint256 {
buf := io.NewBufBinWriter()
s.EncodeBinary(buf.BinWriter)
return hash.Sha256(buf.Bytes())
}
// Equals checks if s == other.
func (s *MPTRootBase) Equals(other *MPTRootBase) bool {
return s.Version == other.Version && s.Index == other.Index &&

View file

@ -2,6 +2,7 @@ package storage
import (
"github.com/syndtr/goleveldb/leveldb"
"github.com/syndtr/goleveldb/leveldb/filter"
"github.com/syndtr/goleveldb/leveldb/opt"
"github.com/syndtr/goleveldb/leveldb/util"
)
@ -21,8 +22,9 @@ type LevelDBStore struct {
// NewLevelDBStore returns a new LevelDBStore object that will
// initialize the database found at the given path.
func NewLevelDBStore(cfg LevelDBOptions) (*LevelDBStore, error) {
var opts *opt.Options // should be exposed via LevelDBOptions if anything needed
var opts = new(opt.Options) // should be exposed via LevelDBOptions if anything needed
opts.Filter = filter.NewBloomFilter(10)
db, err := leveldb.OpenFile(cfg.DataDirectoryPath, opts)
if err != nil {
return nil, err

View file

@ -110,8 +110,8 @@ func (t *Transaction) Hash() util.Uint256 {
return t.hash
}
// VerificationHash returns the hash of the transaction used to verify it.
func (t *Transaction) VerificationHash() util.Uint256 {
// GetSignedHash returns a hash of the transaction used to verify it.
func (t *Transaction) GetSignedHash() util.Uint256 {
if t.verificationHash.Equals(util.Uint256{}) {
if t.createHash() != nil {
panic("failed to compute hash!")

View file

@ -128,15 +128,19 @@ func (p *PrivateKey) GetScriptHash() util.Uint160 {
return pk.GetScriptHash()
}
// Sign signs arbitrary length data using the private key.
// Sign signs arbitrary length data using the private key. It uses SHA256 to
// calculate hash and then SignHash to create a signature (so you can save on
// hash calculation if you already have it).
func (p *PrivateKey) Sign(data []byte) []byte {
var (
privateKey = &p.PrivateKey
digest = sha256.Sum256(data)
)
var digest = sha256.Sum256(data)
r, s := rfc6979.SignECDSA(privateKey, digest[:], sha256.New)
return getSignatureSlice(privateKey.Curve, r, s)
return p.SignHash(digest)
}
// SignHash signs particular hash the private key.
func (p *PrivateKey) SignHash(digest util.Uint256) []byte {
r, s := rfc6979.SignECDSA(&p.PrivateKey, digest[:], sha256.New)
return getSignatureSlice(p.PrivateKey.Curve, r, s)
}
func getSignatureSlice(curve elliptic.Curve, r, s *big.Int) []byte {

View file

@ -11,6 +11,7 @@ import (
"math/big"
"github.com/btcsuite/btcd/btcec"
lru "github.com/hashicorp/golang-lru"
"github.com/nspcc-dev/neo-go/pkg/core/interop/interopnames"
"github.com/nspcc-dev/neo-go/pkg/crypto/hash"
"github.com/nspcc-dev/neo-go/pkg/encoding/address"
@ -105,13 +106,31 @@ func NewPublicKeyFromString(s string) (*PublicKey, error) {
return NewPublicKeyFromBytes(b, elliptic.P256())
}
// keycache is a simple lru cache for P256 keys that avoids Y calculation overhead
// for known keys.
var keycache *lru.Cache
func init() {
// Less than 100K, probably enough for our purposes.
keycache, _ = lru.New(1024)
}
// NewPublicKeyFromBytes returns public key created from b using given EC.
func NewPublicKeyFromBytes(b []byte, curve elliptic.Curve) (*PublicKey, error) {
pubKey := new(PublicKey)
var pubKey *PublicKey
cachedKey, ok := keycache.Get(string(b))
if ok {
pubKey = cachedKey.(*PublicKey)
if pubKey.Curve == curve {
return pubKey, nil
}
}
pubKey = new(PublicKey)
pubKey.Curve = curve
if err := pubKey.DecodeBytes(b); err != nil {
return nil, err
}
keycache.Add(string(b), pubKey)
return pubKey, nil
}

View file

@ -63,6 +63,10 @@ func TestNewPublicKeyFromBytes(t *testing.T) {
pub, err := NewPublicKeyFromBytes(b, elliptic.P256())
require.NoError(t, err)
require.Equal(t, priv.PublicKey(), pub)
// Test cached access
pub2, err := NewPublicKeyFromBytes(b, elliptic.P256())
require.NoError(t, err)
require.Same(t, pub, pub2)
}
func TestDecodeFromString(t *testing.T) {

View file

@ -1,8 +1,11 @@
package crypto
import "github.com/nspcc-dev/neo-go/pkg/util"
// Verifiable represents an object which can be verified.
type Verifiable interface {
GetSignedPart() []byte
GetSignedHash() util.Uint256
}
// VerifiableDecodable represents an object which can be both verified and

View file

@ -213,7 +213,8 @@ func (m *Message) tryCompressPayload() error {
compressedPayload := buf.Bytes()
if m.Flags&Compressed == 0 {
switch m.Payload.(type) {
case *payload.Headers, *payload.MerkleBlock, *payload.NullPayload:
case *payload.Headers, *payload.MerkleBlock, *payload.NullPayload,
*payload.Inventory:
break
default:
size := len(compressedPayload)

View file

@ -795,22 +795,33 @@ func (s *Server) requestTx(hashes ...util.Uint256) {
return
}
msg := NewMessage(CMDGetData, payload.NewInventory(payload.TXType, hashes))
// It's high priority because it directly affects consensus process,
// even though it's getdata.
s.broadcastHPMessage(msg)
for i := 0; i < len(hashes)/payload.MaxHashesCount; i++ {
start := i * payload.MaxHashesCount
stop := (i + 1) * payload.MaxHashesCount
if stop < len(hashes) {
stop = len(hashes)
}
msg := NewMessage(CMDGetData, payload.NewInventory(payload.TXType, hashes[start:stop]))
// It's high priority because it directly affects consensus process,
// even though it's getdata.
s.broadcastHPMessage(msg)
}
}
// iteratePeersWithSendMsg sends given message to all peers using two functions
// passed, one is to send the message and the other is to filtrate peers (the
// peer is considered invalid if it returns false).
func (s *Server) iteratePeersWithSendMsg(msg *Message, send func(Peer, []byte) error, peerOK func(Peer) bool) {
// Get a copy of s.peers to avoid holding a lock while sending.
peers := s.Peers()
if len(peers) == 0 {
return
}
pkt, err := msg.Bytes()
if err != nil {
return
}
// Get a copy of s.peers to avoid holding a lock while sending.
for peer := range s.Peers() {
for peer := range peers {
if peerOK != nil && !peerOK(peer) {
continue
}

View file

@ -2,8 +2,6 @@ package response
import (
"encoding/json"
"github.com/nspcc-dev/neo-go/pkg/rpc/response/result"
)
// Header is a generic JSON-RPC 2.0 response header (ID and JSON-RPC version).
@ -26,10 +24,11 @@ type Raw struct {
Result json.RawMessage `json:"result,omitempty"`
}
// GetRawTx represents verbose output of `getrawtransaction` RPC call.
type GetRawTx struct {
// Abstract represents abstract JSON-RPC 2.0 response, it differs from Raw in
// that Result field is an interface here.
type Abstract struct {
HeaderAndError
Result *result.TransactionOutputRaw `json:"result"`
Result interface{} `json:"result,omitempty"`
}
// Notification is a type used to represent wire format of events, they're

View file

@ -230,7 +230,7 @@ func (s *Server) handleHTTPRequest(w http.ResponseWriter, httpRequest *http.Requ
s.log.Info("websocket connection upgrade failed", zap.Error(err))
return
}
resChan := make(chan response.Raw)
resChan := make(chan response.Abstract)
subChan := make(chan *websocket.PreparedMessage, notificationBufSize)
subscr := &subscriber{writer: subChan, ws: ws}
s.subsLock.Lock()
@ -262,13 +262,13 @@ func (s *Server) handleHTTPRequest(w http.ResponseWriter, httpRequest *http.Requ
s.writeHTTPServerResponse(req, w, resp)
}
func (s *Server) handleRequest(req *request.In, sub *subscriber) response.Raw {
func (s *Server) handleRequest(req *request.In, sub *subscriber) response.Abstract {
var res interface{}
var resErr *response.Error
reqParams, err := req.Params()
if err != nil {
return s.packResponseToRaw(req, nil, response.NewInvalidParamsError("Problem parsing request parameters", err))
return s.packResponse(req, nil, response.NewInvalidParamsError("Problem parsing request parameters", err))
}
s.log.Debug("processing rpc request",
@ -287,10 +287,10 @@ func (s *Server) handleRequest(req *request.In, sub *subscriber) response.Raw {
res, resErr = handler(s, *reqParams, sub)
}
}
return s.packResponseToRaw(req, res, resErr)
return s.packResponse(req, res, resErr)
}
func (s *Server) handleWsWrites(ws *websocket.Conn, resChan <-chan response.Raw, subChan <-chan *websocket.PreparedMessage) {
func (s *Server) handleWsWrites(ws *websocket.Conn, resChan <-chan response.Abstract, subChan <-chan *websocket.PreparedMessage) {
pingTicker := time.NewTicker(wsPingPeriod)
eventloop:
for {
@ -337,7 +337,7 @@ drainloop:
}
}
func (s *Server) handleWsReads(ws *websocket.Conn, resChan chan<- response.Raw, subscr *subscriber) {
func (s *Server) handleWsReads(ws *websocket.Conn, resChan chan<- response.Abstract, subscr *subscriber) {
ws.SetReadLimit(wsReadLimit)
ws.SetReadDeadline(time.Now().Add(wsPongLimit))
ws.SetPongHandler(func(string) error { ws.SetReadDeadline(time.Now().Add(wsPongLimit)); return nil })
@ -712,7 +712,7 @@ func (s *Server) getStorage(ps request.Params) (interface{}, *response.Error) {
item := s.chain.GetStorageItem(id, key)
if item == nil {
return nil, nil
return "", nil
}
return hex.EncodeToString(item.Value), nil
@ -1250,8 +1250,8 @@ func (s *Server) blockHeightFromParam(param *request.Param) (int, *response.Erro
return num, nil
}
func (s *Server) packResponseToRaw(r *request.In, result interface{}, respErr *response.Error) response.Raw {
resp := response.Raw{
func (s *Server) packResponse(r *request.In, result interface{}, respErr *response.Error) response.Abstract {
resp := response.Abstract{
HeaderAndError: response.HeaderAndError{
Header: response.Header{
JSONRPC: r.JSONRPC,
@ -1262,15 +1262,7 @@ func (s *Server) packResponseToRaw(r *request.In, result interface{}, respErr *r
if respErr != nil {
resp.Error = respErr
} else {
resJSON, err := json.Marshal(result)
if err != nil {
s.log.Error("failed to marshal result",
zap.Error(err),
zap.String("method", r.Method))
resp.Error = response.NewInternalServerError("failed to encode result", err)
} else {
resp.Result = resJSON
}
resp.Result = result
}
return resp
}
@ -1292,11 +1284,11 @@ func (s *Server) logRequestError(r *request.In, jsonErr *response.Error) {
// writeHTTPErrorResponse writes an error response to the ResponseWriter.
func (s *Server) writeHTTPErrorResponse(r *request.In, w http.ResponseWriter, jsonErr *response.Error) {
resp := s.packResponseToRaw(r, nil, jsonErr)
resp := s.packResponse(r, nil, jsonErr)
s.writeHTTPServerResponse(r, w, resp)
}
func (s *Server) writeHTTPServerResponse(r *request.In, w http.ResponseWriter, resp response.Raw) {
func (s *Server) writeHTTPServerResponse(r *request.In, w http.ResponseWriter, resp response.Abstract) {
// Errors can happen in many places and we can only catch ALL of them here.
if resp.Error != nil {
s.logRequestError(r, resp.Error)

View file

@ -91,6 +91,10 @@ func (fs *FeerStub) FeePerByte() int64 {
return 0
}
func (fs *FeerStub) BlockHeight() uint32 {
return 0
}
func (fs *FeerStub) GetUtilityTokenBalance(acc util.Uint160) *big.Int {
return big.NewInt(1000000 * native.GASFactor)
}