From c9b1d359d8814e1711e40935d4c7b558d6be8679 Mon Sep 17 00:00:00 2001 From: Roman Khimov Date: Thu, 7 May 2020 21:46:28 +0300 Subject: [PATCH 01/20] core: ensure we produce correct blocks for tests Check for correct merkle root generation. --- pkg/core/helper_test.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/pkg/core/helper_test.go b/pkg/core/helper_test.go index 05f67e7e9..9ff510f05 100644 --- a/pkg/core/helper_test.go +++ b/pkg/core/helper_test.go @@ -71,7 +71,10 @@ func newBlock(cfg config.ProtocolConfiguration, index uint32, prev util.Uint256, }, Transactions: txs, } - _ = b.RebuildMerkleRoot() + err := b.RebuildMerkleRoot() + if err != nil { + panic(err) + } b.Script.InvocationScript = testchain.Sign(b.GetSignedPart()) return b From f2033b5e54d91a5a0bd5a9c0fac7c8f9e537401b Mon Sep 17 00:00:00 2001 From: Roman Khimov Date: Thu, 7 May 2020 21:48:45 +0300 Subject: [PATCH 02/20] wallet: check for t.GetSignedPart() result correctness It can return nil easily and signing that is a big mistake. --- pkg/wallet/account.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pkg/wallet/account.go b/pkg/wallet/account.go index 2236e7216..22a440877 100644 --- a/pkg/wallet/account.go +++ b/pkg/wallet/account.go @@ -132,6 +132,9 @@ func (a *Account) SignTx(t *transaction.Transaction) error { return errors.New("account is not unlocked") } data := t.GetSignedPart() + if data == nil { + return errors.New("failed to get transaction's signed part") + } sign := a.privateKey.Sign(data) t.Scripts = append(t.Scripts, transaction.Witness{ From ac40d357f01dd795aff345844298a6db9cd5e843 Mon Sep 17 00:00:00 2001 From: Roman Khimov Date: Thu, 7 May 2020 22:04:10 +0300 Subject: [PATCH 03/20] core: improve documentation a little --- pkg/core/blockchain.go | 10 +++++++--- pkg/core/doc.go | 5 +++++ 2 files changed, 12 insertions(+), 3 deletions(-) create mode 100644 pkg/core/doc.go diff --git a/pkg/core/blockchain.go b/pkg/core/blockchain.go index 0ef370b8f..f7c93eb9d 100644 --- a/pkg/core/blockchain.go +++ b/pkg/core/blockchain.go @@ -64,7 +64,9 @@ var ( persistInterval = 1 * time.Second ) -// Blockchain represents the blockchain. +// Blockchain represents the blockchain. It maintans internal state representing +// the state of the ledger that can be accessed in various ways and changed by +// adding new blocks or headers. type Blockchain struct { config config.ProtocolConfiguration @@ -130,7 +132,8 @@ type Blockchain struct { type headersOpFunc func(headerList *HeaderHashList) // NewBlockchain returns a new blockchain object the will use the -// given Store as its underlying storage. +// given Store as its underlying storage. For it to work correctly you need +// to spawn a goroutine for its Run method after this initialization. func NewBlockchain(s storage.Store, cfg config.ProtocolConfiguration, log *zap.Logger) (*Blockchain, error) { if log == nil { return nil, errors.New("empty logger") @@ -269,7 +272,8 @@ func (bc *Blockchain) init() error { return nil } -// Run runs chain loop. +// Run runs chain loop, it needs to be run as goroutine and executing it is +// critical for correct Blockchain operation. func (bc *Blockchain) Run() { persistTimer := time.NewTimer(persistInterval) defer func() { diff --git a/pkg/core/doc.go b/pkg/core/doc.go new file mode 100644 index 000000000..b0364db7f --- /dev/null +++ b/pkg/core/doc.go @@ -0,0 +1,5 @@ +/* +Package core implements Neo ledger functionality. +It's built around the Blockchain structure that maintains state of the ledger. +*/ +package core From 3ca71d5e8db6369b09c59af1da7232b4683b857d Mon Sep 17 00:00:00 2001 From: Roman Khimov Date: Tue, 12 May 2020 17:20:41 +0300 Subject: [PATCH 04/20] core: add Blockchain event subscription mechanism A deep internal part of #895. Blockchainer interface is also extended for various uses of these methods. --- pkg/core/blockchain.go | 164 ++++++++++++++++++++++++++ pkg/core/blockchain_test.go | 108 +++++++++++++++++ pkg/core/blockchainer/blockchainer.go | 8 ++ pkg/core/doc.go | 24 ++++ pkg/network/helper_test.go | 26 ++++ 5 files changed, 330 insertions(+) diff --git a/pkg/core/blockchain.go b/pkg/core/blockchain.go index f7c93eb9d..54de3f1bd 100644 --- a/pkg/core/blockchain.go +++ b/pkg/core/blockchain.go @@ -127,6 +127,20 @@ type Blockchain struct { lastBatch *storage.MemBatch contracts native.Contracts + + // Notification subsystem. + events chan bcEvent + subCh chan interface{} + unsubCh chan interface{} +} + +// bcEvent is an internal event generated by the Blockchain and then +// broadcasted to other parties. It joins the new block and associated +// invocation logs, all the other events visible from outside can be produced +// from this combination. +type bcEvent struct { + block *block.Block + appExecResults []*state.AppExecResult } type headersOpFunc func(headerList *HeaderHashList) @@ -169,6 +183,9 @@ func NewBlockchain(s storage.Store, cfg config.ProtocolConfiguration, log *zap.L memPool: mempool.NewMemPool(cfg.MemPoolSize), keyCache: make(map[util.Uint160]map[string]*keys.PublicKey), log: log, + events: make(chan bcEvent), + subCh: make(chan interface{}), + unsubCh: make(chan interface{}), generationAmount: genAmount, decrementInterval: decrementInterval, @@ -286,6 +303,7 @@ func (bc *Blockchain) Run() { } close(bc.runToExitCh) }() + go bc.notificationDispatcher() for { select { case <-bc.stopCh: @@ -305,6 +323,82 @@ func (bc *Blockchain) Run() { } } +// notificationDispatcher manages subscription to events and broadcasts new events. +func (bc *Blockchain) notificationDispatcher() { + var ( + // These are just sets of subscribers, though modelled as maps + // for ease of management (not a lot of subscriptions is really + // expected, but maps are convenient for adding/deleting elements). + blockFeed = make(map[chan<- *block.Block]bool) + txFeed = make(map[chan<- *transaction.Transaction]bool) + notificationFeed = make(map[chan<- *state.NotificationEvent]bool) + executionFeed = make(map[chan<- *state.AppExecResult]bool) + ) + for { + select { + case <-bc.stopCh: + return + case sub := <-bc.subCh: + switch ch := sub.(type) { + case chan<- *block.Block: + blockFeed[ch] = true + case chan<- *transaction.Transaction: + txFeed[ch] = true + case chan<- *state.NotificationEvent: + notificationFeed[ch] = true + case chan<- *state.AppExecResult: + executionFeed[ch] = true + default: + panic(fmt.Sprintf("bad subscription: %T", sub)) + } + case unsub := <-bc.unsubCh: + switch ch := unsub.(type) { + case chan<- *block.Block: + delete(blockFeed, ch) + case chan<- *transaction.Transaction: + delete(txFeed, ch) + case chan<- *state.NotificationEvent: + delete(notificationFeed, ch) + case chan<- *state.AppExecResult: + delete(executionFeed, ch) + default: + panic(fmt.Sprintf("bad unsubscription: %T", unsub)) + } + case event := <-bc.events: + // We don't want to waste time looping through transactions when there are no + // subscribers. + if len(txFeed) != 0 || len(notificationFeed) != 0 || len(executionFeed) != 0 { + var aerIdx int + for _, tx := range event.block.Transactions { + if tx.Type == transaction.InvocationType { + aer := event.appExecResults[aerIdx] + if !aer.TxHash.Equals(tx.Hash()) { + panic("inconsistent application execution results") + } + aerIdx++ + for ch := range executionFeed { + ch <- aer + } + if aer.VMState == "HALT" { + for i := range aer.Events { + for ch := range notificationFeed { + ch <- &aer.Events[i] + } + } + } + } + for ch := range txFeed { + ch <- tx + } + } + } + for ch := range blockFeed { + ch <- event.block + } + } + } +} + // Close stops Blockchain's internal loop, syncs changes to persistent storage // and closes it. The Blockchain is no longer functional after the call to Close. func (bc *Blockchain) Close() { @@ -468,6 +562,7 @@ func (bc *Blockchain) getSystemFeeAmount(h util.Uint256) uint32 { // and all tests are in place, we can make a more optimized and cleaner implementation. func (bc *Blockchain) storeBlock(block *block.Block) error { cache := dao.NewCached(bc.dao) + appExecResults := make([]*state.AppExecResult, 0, len(block.Transactions)) fee := bc.getSystemFeeAmount(block.PrevHash) for _, tx := range block.Transactions { fee += uint32(tx.SystemFee.IntegralValue()) @@ -700,6 +795,7 @@ func (bc *Blockchain) storeBlock(block *block.Block) error { Stack: v.Estack().ToContractParameters(), Events: systemInterop.Notifications, } + appExecResults = append(appExecResults, aer) err = cache.PutAppExecResult(aer) if err != nil { return errors.Wrap(err, "failed to Store notifications") @@ -728,6 +824,12 @@ func (bc *Blockchain) storeBlock(block *block.Block) error { atomic.StoreUint32(&bc.blockHeight, block.Index) updateBlockHeightMetric(block.Index) bc.memPool.RemoveStale(bc.isTxStillRelevant, bc) + // Genesis block is stored when Blockchain is not yet running, so there + // is no one to read this event. And it doesn't make much sense as event + // anyway. + if block.Index != 0 { + bc.events <- bcEvent{block, appExecResults} + } return nil } @@ -1054,6 +1156,68 @@ func (bc *Blockchain) GetConfig() config.ProtocolConfiguration { return bc.config } +// SubscribeForBlocks adds given channel to new block event broadcasting, so when +// there is a new block added to the chain you'll receive it via this channel. +// Make sure it's read from regularly as not reading these events might affect +// other Blockchain functions. +func (bc *Blockchain) SubscribeForBlocks(ch chan<- *block.Block) { + bc.subCh <- ch +} + +// SubscribeForTransactions adds given channel to new transaction event +// broadcasting, so when there is a new transaction added to the chain (in a +// block) you'll receive it via this channel. Make sure it's read from regularly +// as not reading these events might affect other Blockchain functions. +func (bc *Blockchain) SubscribeForTransactions(ch chan<- *transaction.Transaction) { + bc.subCh <- ch +} + +// SubscribeForNotifications adds given channel to new notifications event +// broadcasting, so when an in-block transaction execution generates a +// notification you'll receive it via this channel. Only notifications from +// successful transactions are broadcasted, if you're interested in failed +// transactions use SubscribeForExecutions instead. Make sure this channel is +// read from regularly as not reading these events might affect other Blockchain +// functions. +func (bc *Blockchain) SubscribeForNotifications(ch chan<- *state.NotificationEvent) { + bc.subCh <- ch +} + +// SubscribeForExecutions adds given channel to new transaction execution event +// broadcasting, so when an in-block transaction execution happens you'll receive +// the result of it via this channel. Make sure it's read from regularly as not +// reading these events might affect other Blockchain functions. +func (bc *Blockchain) SubscribeForExecutions(ch chan<- *state.AppExecResult) { + bc.subCh <- ch +} + +// UnsubscribeFromBlocks unsubscribes given channel from new block notifications, +// you can close it afterwards. Passing non-subscribed channel is a no-op. +func (bc *Blockchain) UnsubscribeFromBlocks(ch chan<- *block.Block) { + bc.unsubCh <- ch +} + +// UnsubscribeFromTransactions unsubscribes given channel from new transaction +// notifications, you can close it afterwards. Passing non-subscribed channel is +// a no-op. +func (bc *Blockchain) UnsubscribeFromTransactions(ch chan<- *transaction.Transaction) { + bc.unsubCh <- ch +} + +// UnsubscribeFromNotifications unsubscribes given channel from new +// execution-generated notifications, you can close it afterwards. Passing +// non-subscribed channel is a no-op. +func (bc *Blockchain) UnsubscribeFromNotifications(ch chan<- *state.NotificationEvent) { + bc.unsubCh <- ch +} + +// UnsubscribeFromExecutions unsubscribes given channel from new execution +// notifications, you can close it afterwards. Passing non-subscribed channel is +// a no-op. +func (bc *Blockchain) UnsubscribeFromExecutions(ch chan<- *state.AppExecResult) { + bc.unsubCh <- ch +} + // CalculateClaimable calculates the amount of GAS which can be claimed for a transaction with value. // First return value is GAS generated between startHeight and endHeight. // Second return value is GAS returned from accumulated SystemFees between startHeight and endHeight. diff --git a/pkg/core/blockchain_test.go b/pkg/core/blockchain_test.go index a106d4fba..77a5d697e 100644 --- a/pkg/core/blockchain_test.go +++ b/pkg/core/blockchain_test.go @@ -2,12 +2,17 @@ package core import ( "testing" + "time" "github.com/nspcc-dev/neo-go/pkg/core/block" + "github.com/nspcc-dev/neo-go/pkg/core/state" "github.com/nspcc-dev/neo-go/pkg/core/storage" "github.com/nspcc-dev/neo-go/pkg/core/transaction" "github.com/nspcc-dev/neo-go/pkg/crypto/hash" + "github.com/nspcc-dev/neo-go/pkg/io" "github.com/nspcc-dev/neo-go/pkg/util" + "github.com/nspcc-dev/neo-go/pkg/vm/emit" + "github.com/nspcc-dev/neo-go/pkg/vm/opcode" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -234,3 +239,106 @@ func TestClose(t *testing.T) { // This should never be executed. assert.Nil(t, t) } + +func TestSubscriptions(t *testing.T) { + // We use buffering here as a substitute for reader goroutines, events + // get queued up and we read them one by one here. + const chBufSize = 16 + blockCh := make(chan *block.Block, chBufSize) + txCh := make(chan *transaction.Transaction, chBufSize) + notificationCh := make(chan *state.NotificationEvent, chBufSize) + executionCh := make(chan *state.AppExecResult, chBufSize) + + bc := newTestChain(t) + bc.SubscribeForBlocks(blockCh) + bc.SubscribeForTransactions(txCh) + bc.SubscribeForNotifications(notificationCh) + bc.SubscribeForExecutions(executionCh) + + assert.Empty(t, notificationCh) + assert.Empty(t, executionCh) + assert.Empty(t, blockCh) + assert.Empty(t, txCh) + + blocks, err := bc.genBlocks(1) + require.NoError(t, err) + require.Eventually(t, func() bool { return len(blockCh) != 0 }, time.Second, 10*time.Millisecond) + assert.Empty(t, notificationCh) + assert.Empty(t, executionCh) + assert.Empty(t, txCh) + + b := <-blockCh + assert.Equal(t, blocks[0], b) + assert.Empty(t, blockCh) + + script := io.NewBufBinWriter() + emit.Bytes(script.BinWriter, []byte("yay!")) + emit.Syscall(script.BinWriter, "Neo.Runtime.Notify") + require.NoError(t, script.Err) + txGood1 := transaction.NewInvocationTX(script.Bytes(), 0) + txGood1.Sender = neoOwner + txGood1.Nonce = 1 + txGood1.ValidUntilBlock = 100500 + require.NoError(t, signTx(bc, txGood1)) + + // Reset() reuses the script buffer and we need to keep scripts. + script = io.NewBufBinWriter() + emit.Bytes(script.BinWriter, []byte("nay!")) + emit.Syscall(script.BinWriter, "Neo.Runtime.Notify") + emit.Opcode(script.BinWriter, opcode.THROW) + require.NoError(t, script.Err) + txBad := transaction.NewInvocationTX(script.Bytes(), 0) + txBad.Sender = neoOwner + txBad.Nonce = 2 + txBad.ValidUntilBlock = 100500 + require.NoError(t, signTx(bc, txBad)) + + script = io.NewBufBinWriter() + emit.Bytes(script.BinWriter, []byte("yay! yay! yay!")) + emit.Syscall(script.BinWriter, "Neo.Runtime.Notify") + require.NoError(t, script.Err) + txGood2 := transaction.NewInvocationTX(script.Bytes(), 0) + txGood2.Sender = neoOwner + txGood2.Nonce = 3 + txGood2.ValidUntilBlock = 100500 + require.NoError(t, signTx(bc, txGood2)) + + invBlock := newBlock(bc.config, bc.BlockHeight()+1, bc.CurrentHeaderHash(), txGood1, txBad, txGood2) + require.NoError(t, bc.AddBlock(invBlock)) + + require.Eventually(t, func() bool { + return len(blockCh) != 0 && len(txCh) != 0 && + len(notificationCh) != 0 && len(executionCh) != 0 + }, time.Second, 10*time.Millisecond) + + b = <-blockCh + require.Equal(t, invBlock, b) + assert.Empty(t, blockCh) + + // Follow in-block transaction order. + for _, txExpected := range invBlock.Transactions { + tx := <-txCh + require.Equal(t, txExpected, tx) + if txExpected.Type == transaction.InvocationType { + exec := <-executionCh + require.Equal(t, tx.Hash(), exec.TxHash) + if exec.VMState == "HALT" { + notif := <-notificationCh + inv := tx.Data.(*transaction.InvocationTX) + require.Equal(t, hash.Hash160(inv.Script), notif.ScriptHash) + } + } + } + assert.Empty(t, txCh) + assert.Empty(t, notificationCh) + assert.Empty(t, executionCh) + + bc.UnsubscribeFromBlocks(blockCh) + bc.UnsubscribeFromTransactions(txCh) + bc.UnsubscribeFromNotifications(notificationCh) + bc.UnsubscribeFromExecutions(executionCh) + + // Ensure that new blocks are processed correctly after unsubscription. + _, err = bc.genBlocks(2 * chBufSize) + require.NoError(t, err) +} diff --git a/pkg/core/blockchainer/blockchainer.go b/pkg/core/blockchainer/blockchainer.go index d749ec995..dacb0f2fd 100644 --- a/pkg/core/blockchainer/blockchainer.go +++ b/pkg/core/blockchainer/blockchainer.go @@ -47,6 +47,14 @@ type Blockchainer interface { References(t *transaction.Transaction) ([]transaction.InOut, error) mempool.Feer // fee interface PoolTx(*transaction.Transaction) error + SubscribeForBlocks(ch chan<- *block.Block) + SubscribeForExecutions(ch chan<- *state.AppExecResult) + SubscribeForNotifications(ch chan<- *state.NotificationEvent) + SubscribeForTransactions(ch chan<- *transaction.Transaction) VerifyTx(*transaction.Transaction, *block.Block) error GetMemPool() *mempool.Pool + UnsubscribeFromBlocks(ch chan<- *block.Block) + UnsubscribeFromExecutions(ch chan<- *state.AppExecResult) + UnsubscribeFromNotifications(ch chan<- *state.NotificationEvent) + UnsubscribeFromTransactions(ch chan<- *transaction.Transaction) } diff --git a/pkg/core/doc.go b/pkg/core/doc.go index b0364db7f..c459ea132 100644 --- a/pkg/core/doc.go +++ b/pkg/core/doc.go @@ -1,5 +1,29 @@ /* Package core implements Neo ledger functionality. It's built around the Blockchain structure that maintains state of the ledger. + +Events + +You can subscribe to Blockchain events using a set of Subscribe and Unsubscribe +methods. These methods accept channels that will be used to send appropriate +events, so you can control buffering. Channels are never closed by Blockchain, +you can close them after unsubscription. + +Unlike RPC-level subscriptions these don't allow event filtering because it +doesn't improve overall efficiency much (when you're using Blockchain you're +in the same process with it and filtering on your side is not that different +from filtering on Blockchain side). + +The same level of ordering guarantees as with RPC subscriptions is provided, +albeit for a set of event channels, so at first transaction execution is +announced via appropriate channels, then followed by notifications generated +during this execution, then followed by transaction announcement and then +followed by block announcement. Transaction announcements are ordered the same +way they're stored in the block. + +Be careful using these subscriptions, this mechanism is not intended to be used +by lots of subscribers and failing to read from event channels can affect +other Blockchain operations. + */ package core diff --git a/pkg/network/helper_test.go b/pkg/network/helper_test.go index 411afc88d..b29764c5b 100644 --- a/pkg/network/helper_test.go +++ b/pkg/network/helper_test.go @@ -145,10 +145,36 @@ func (chain testChain) PoolTx(*transaction.Transaction) error { panic("TODO") } +func (chain testChain) SubscribeForBlocks(ch chan<- *block.Block) { + panic("TODO") +} +func (chain testChain) SubscribeForExecutions(ch chan<- *state.AppExecResult) { + panic("TODO") +} +func (chain testChain) SubscribeForNotifications(ch chan<- *state.NotificationEvent) { + panic("TODO") +} +func (chain testChain) SubscribeForTransactions(ch chan<- *transaction.Transaction) { + panic("TODO") +} + func (chain testChain) VerifyTx(*transaction.Transaction, *block.Block) error { panic("TODO") } +func (chain testChain) UnsubscribeFromBlocks(ch chan<- *block.Block) { + panic("TODO") +} +func (chain testChain) UnsubscribeFromExecutions(ch chan<- *state.AppExecResult) { + panic("TODO") +} +func (chain testChain) UnsubscribeFromNotifications(ch chan<- *state.NotificationEvent) { + panic("TODO") +} +func (chain testChain) UnsubscribeFromTransactions(ch chan<- *transaction.Transaction) { + panic("TODO") +} + type testDiscovery struct{} func (d testDiscovery) BackFill(addrs ...string) {} From 462022bbdd2b4b0354d6d027921204c38e32d242 Mon Sep 17 00:00:00 2001 From: Roman Khimov Date: Thu, 7 May 2020 22:45:06 +0300 Subject: [PATCH 05/20] consensus: remove OnNewBlock(), use Blockchain subscription Get new blocks directly from the Blockchain. It may lead to some duplications (as we'll also receive our own blocks), but at the same time it's more correct, because technically we can also get blocks via other means besides network server like RPC (submitblock call). And it simplifies network server at the same time. --- pkg/consensus/consensus.go | 36 +++++++++++------------------------- pkg/network/server.go | 4 +--- 2 files changed, 12 insertions(+), 28 deletions(-) diff --git a/pkg/consensus/consensus.go b/pkg/consensus/consensus.go index 6452931ad..8f8bbd7d4 100644 --- a/pkg/consensus/consensus.go +++ b/pkg/consensus/consensus.go @@ -42,9 +42,6 @@ type Service interface { OnTransaction(tx *transaction.Transaction) // GetPayload returns Payload with specified hash if it is present in the local cache. GetPayload(h util.Uint256) *Payload - // OnNewBlock notifies consensus service that there is a new block in - // the chain (without explicitly passing it to the service). - OnNewBlock() } type service struct { @@ -62,7 +59,7 @@ type service struct { transactions chan *transaction.Transaction // blockEvents is used to pass a new block event to the consensus // process. - blockEvents chan struct{} + blockEvents chan *coreb.Block lastProposal []util.Uint256 wallet *wallet.Wallet } @@ -107,7 +104,7 @@ func NewService(cfg Config) (Service, error) { messages: make(chan Payload, 100), transactions: make(chan *transaction.Transaction, 100), - blockEvents: make(chan struct{}, 1), + blockEvents: make(chan *coreb.Block, 1), } if cfg.Wallet == nil { @@ -164,7 +161,7 @@ var ( func (s *service) Start() { s.dbft.Start() - + s.Chain.SubscribeForBlocks(s.blockEvents) go s.eventLoop() } @@ -204,11 +201,14 @@ func (s *service) eventLoop() { s.dbft.OnReceive(&msg) case tx := <-s.transactions: s.dbft.OnTransaction(tx) - case <-s.blockEvents: - s.log.Debug("new block in the chain", - zap.Uint32("dbft index", s.dbft.BlockIndex), - zap.Uint32("chain index", s.Chain.BlockHeight())) - s.dbft.InitializeConsensus(0) + case b := <-s.blockEvents: + // We also receive our own blocks here, so check for index. + if b.Index >= s.dbft.BlockIndex { + s.log.Debug("new block in the chain", + zap.Uint32("dbft index", s.dbft.BlockIndex), + zap.Uint32("chain index", s.Chain.BlockHeight())) + s.dbft.InitializeConsensus(0) + } } } } @@ -287,20 +287,6 @@ func (s *service) OnTransaction(tx *transaction.Transaction) { } } -// OnNewBlock notifies consensus process that there is a new block in the chain -// and dbft should probably be reinitialized. -func (s *service) OnNewBlock() { - if s.dbft != nil { - // If there is something in the queue already, the second - // consecutive event doesn't make much sense (reinitializing - // dbft twice doesn't improve it in any way). - select { - case s.blockEvents <- struct{}{}: - default: - } - } -} - // GetPayload returns payload stored in cache. func (s *service) GetPayload(h util.Uint256) *Payload { p := s.cache.Get(h) diff --git a/pkg/network/server.go b/pkg/network/server.go index 1ecc7782d..0031ec877 100644 --- a/pkg/network/server.go +++ b/pkg/network/server.go @@ -103,9 +103,7 @@ func NewServer(config ServerConfig, chain blockchainer.Blockchainer, log *zap.Lo transactions: make(chan *transaction.Transaction, 64), } s.bQueue = newBlockQueue(maxBlockBatch, chain, log, func(b *block.Block) { - if s.consensusStarted.Load() { - s.consensus.OnNewBlock() - } else { + if !s.consensusStarted.Load() { s.tryStartConsensus() } s.relayBlock(b) From b7d2b659b4805cbd3a8c42cff4942eae2f5b6d0b Mon Sep 17 00:00:00 2001 From: Roman Khimov Date: Thu, 7 May 2020 23:00:38 +0300 Subject: [PATCH 06/20] network: get blocks directly from the chain for rebroadcasting Simplify network<->consensus relations, also broadcast blocks received by other means like RPC. --- pkg/consensus/consensus.go | 5 ----- pkg/network/server.go | 40 ++++++++++++++++++++++++-------------- 2 files changed, 25 insertions(+), 20 deletions(-) diff --git a/pkg/consensus/consensus.go b/pkg/consensus/consensus.go index 8f8bbd7d4..190f3577c 100644 --- a/pkg/consensus/consensus.go +++ b/pkg/consensus/consensus.go @@ -71,9 +71,6 @@ type Config struct { // Broadcast is a callback which is called to notify server // about new consensus payload to sent. Broadcast func(p *Payload) - // RelayBlock is a callback that is called to notify server - // about the new block that needs to be broadcasted. - RelayBlock func(b *coreb.Block) // Chain is a core.Blockchainer instance. Chain blockchainer.Blockchainer // RequestTx is a callback to which will be called @@ -348,8 +345,6 @@ func (s *service) processBlock(b block.Block) { if _, errget := s.Chain.GetBlock(bb.Hash()); errget != nil { s.log.Warn("error on add block", zap.Error(err)) } - } else { - s.Config.RelayBlock(bb) } } diff --git a/pkg/network/server.go b/pkg/network/server.go index 0031ec877..c9a5e1038 100644 --- a/pkg/network/server.go +++ b/pkg/network/server.go @@ -106,16 +106,14 @@ func NewServer(config ServerConfig, chain blockchainer.Blockchainer, log *zap.Lo if !s.consensusStarted.Load() { s.tryStartConsensus() } - s.relayBlock(b) }) srv, err := consensus.NewService(consensus.Config{ - Logger: log, - Broadcast: s.handleNewPayload, - RelayBlock: s.relayBlock, - Chain: chain, - RequestTx: s.requestTx, - Wallet: config.Wallet, + Logger: log, + Broadcast: s.handleNewPayload, + Chain: chain, + RequestTx: s.requestTx, + Wallet: config.Wallet, TimePerBlock: config.TimePerBlock, }) @@ -171,6 +169,7 @@ func (s *Server) Start(errChan chan error) { s.discovery.BackFill(s.Seeds...) go s.broadcastTxLoop() + go s.relayBlocksLoop() go s.bQueue.run() go s.transport.Accept() setServerAndNodeVersions(s.UserAgent, strconv.FormatUint(uint64(s.id), 10)) @@ -795,14 +794,25 @@ func (s *Server) broadcastHPMessage(msg *Message) { s.iteratePeersWithSendMsg(msg, Peer.EnqueueHPPacket, nil) } -// relayBlock tells all the other connected nodes about the given block. -func (s *Server) relayBlock(b *block.Block) { - msg := NewMessage(CMDInv, payload.NewInventory(payload.BlockType, []util.Uint256{b.Hash()})) - // Filter out nodes that are more current (avoid spamming the network - // during initial sync). - s.iteratePeersWithSendMsg(msg, Peer.EnqueuePacket, func(p Peer) bool { - return p.Handshaked() && p.LastBlockIndex() < b.Index - }) +// relayBlocksLoop subscribes to new blocks in the ledger and broadcasts them +// to the network. Intended to be run as a separate goroutine. +func (s *Server) relayBlocksLoop() { + ch := make(chan *block.Block, 2) // Some buffering to smooth out possible egressing delays. + s.chain.SubscribeForBlocks(ch) + for { + select { + case <-s.quit: + s.chain.UnsubscribeFromBlocks(ch) + return + case b := <-ch: + msg := NewMessage(CMDInv, payload.NewInventory(payload.BlockType, []util.Uint256{b.Hash()})) + // Filter out nodes that are more current (avoid spamming the network + // during initial sync). + s.iteratePeersWithSendMsg(msg, Peer.EnqueuePacket, func(p Peer) bool { + return p.Handshaked() && p.LastBlockIndex() < b.Index + }) + } + } } // verifyAndPoolTX verifies the TX and adds it to the local mempool. From 612500ed96a4f058b24ffd85da170a496b468bf1 Mon Sep 17 00:00:00 2001 From: Roman Khimov Date: Sat, 9 May 2020 23:59:21 +0300 Subject: [PATCH 07/20] rpc/server: start and shutdown Server in tests It will be important for proper subscription testing and it doesn't hurt even though technically we've got two http servers listening after this change (one is a regular Server's http.Server and one is httptest's Server). Reusing rpc.Server would be nice, but it requires some changes to Start sequence to start Listener with net.Listen and then communicate back its resulting Addr. It's not very convenient especially given that no other code needs it, so doing these changes just for a bit cleaner testing seems like and overkill. Update config appropriately. Update Start comment along the way. --- config/protocol.unit_testnet.yml | 3 ++- pkg/rpc/server/server.go | 5 +++-- pkg/rpc/server/server_helper_test.go | 6 ++++-- pkg/rpc/server/server_test.go | 3 ++- 4 files changed, 11 insertions(+), 6 deletions(-) diff --git a/config/protocol.unit_testnet.yml b/config/protocol.unit_testnet.yml index 2c809c523..37424cc3d 100644 --- a/config/protocol.unit_testnet.yml +++ b/config/protocol.unit_testnet.yml @@ -47,9 +47,10 @@ ApplicationConfiguration: AttemptConnPeers: 5 MinPeers: 1 RPC: + Address: 127.0.0.1 Enabled: true EnableCORSWorkaround: false - Port: 20332 + Port: 0 # let the system choose port dynamically Prometheus: Enabled: false #since it's not useful for unit tests. Port: 2112 diff --git a/pkg/rpc/server/server.go b/pkg/rpc/server/server.go index 09c8dd11d..2d7a9e226 100644 --- a/pkg/rpc/server/server.go +++ b/pkg/rpc/server/server.go @@ -123,8 +123,9 @@ func New(chain blockchainer.Blockchainer, conf rpc.Config, coreServer *network.S } } -// Start creates a new JSON-RPC server -// listening on the configured port. +// Start creates a new JSON-RPC server listening on the configured port. It's +// supposed to be run as a separate goroutine (like http.Server's Serve) and it +// returns its errors via given errChan. func (s *Server) Start(errChan chan error) { if !s.config.Enabled { s.log.Info("RPC server is not enabled") diff --git a/pkg/rpc/server/server_helper_test.go b/pkg/rpc/server/server_helper_test.go index b65897ab3..efec61f10 100644 --- a/pkg/rpc/server/server_helper_test.go +++ b/pkg/rpc/server/server_helper_test.go @@ -17,7 +17,7 @@ import ( "go.uber.org/zap/zaptest" ) -func initServerWithInMemoryChain(t *testing.T) (*core.Blockchain, *httptest.Server) { +func initServerWithInMemoryChain(t *testing.T) (*core.Blockchain, *Server, *httptest.Server) { var nBlocks uint32 net := config.ModeUnitTestNet @@ -55,11 +55,13 @@ func initServerWithInMemoryChain(t *testing.T) (*core.Blockchain, *httptest.Serv server, err := network.NewServer(serverConfig, chain, logger) require.NoError(t, err) rpcServer := New(chain, cfg.ApplicationConfiguration.RPC, server, logger) + errCh := make(chan error, 2) + go rpcServer.Start(errCh) handler := http.HandlerFunc(rpcServer.handleHTTPRequest) srv := httptest.NewServer(handler) - return chain, srv + return chain, &rpcServer, srv } type FeerStub struct{} diff --git a/pkg/rpc/server/server_test.go b/pkg/rpc/server/server_test.go index 1d28ab001..b3f64686f 100644 --- a/pkg/rpc/server/server_test.go +++ b/pkg/rpc/server/server_test.go @@ -875,9 +875,10 @@ func TestRPC(t *testing.T) { // calls. Some tests change the chain state, thus we reinitialize the chain from // scratch here. func testRPCProtocol(t *testing.T, doRPCCall func(string, string, *testing.T) []byte) { - chain, httpSrv := initServerWithInMemoryChain(t) + chain, rpcSrv, httpSrv := initServerWithInMemoryChain(t) defer chain.Close() + defer rpcSrv.Shutdown() e := &executor{chain: chain, httpSrv: httpSrv} for method, cases := range rpcTestCases { From 03ecab5eecb16f9a3e289d47a233ab617f4fd49c Mon Sep 17 00:00:00 2001 From: Roman Khimov Date: Mon, 11 May 2020 00:55:15 +0300 Subject: [PATCH 08/20] smartcontract: add JSON marshal/unmarshal for InteropType We actually have to do that in order to answer getapplicationlog requests for transactions that leave some interop items on the stack. It follows the same logic our binary serializer/deserializes does leaving the type and stripping the value (whatever that is). --- pkg/smartcontract/parameter.go | 5 +++++ pkg/smartcontract/parameter_test.go | 34 ++++++++++++++++++++++++----- 2 files changed, 33 insertions(+), 6 deletions(-) diff --git a/pkg/smartcontract/parameter.go b/pkg/smartcontract/parameter.go index 379a75593..5dad09926 100644 --- a/pkg/smartcontract/parameter.go +++ b/pkg/smartcontract/parameter.go @@ -85,6 +85,8 @@ func (p *Parameter) MarshalJSON() ([]byte, error) { case MapType: ppair := p.Value.([]ParameterPair) resultRawValue, resultErr = json.Marshal(ppair) + case InteropInterfaceType: + resultRawValue = []byte("null") default: resultErr = errors.Errorf("Marshaller for type %s not implemented", p.Type) } @@ -166,6 +168,9 @@ func (p *Parameter) UnmarshalJSON(data []byte) (err error) { return } p.Value = h + case InteropInterfaceType: + // stub, ignore value, it can only be null + p.Value = nil default: return errors.Errorf("Unmarshaller for type %s not implemented", p.Type) } diff --git a/pkg/smartcontract/parameter_test.go b/pkg/smartcontract/parameter_test.go index 61d2fd6a6..72ce299a4 100644 --- a/pkg/smartcontract/parameter_test.go +++ b/pkg/smartcontract/parameter_test.go @@ -122,6 +122,13 @@ var marshalJSONTestCases = []struct { }, result: `{"type":"Hash256","value":"0xf037308fa0ab18155bccfc08485468c112409ea5064595699e98c545f245f32d"}`, }, + { + input: Parameter{ + Type: InteropInterfaceType, + Value: nil, + }, + result: `{"type":"InteropInterface","value":null}`, + }, } var marshalJSONErrorCases = []Parameter{ @@ -129,10 +136,6 @@ var marshalJSONErrorCases = []Parameter{ Type: UnknownType, Value: nil, }, - { - Type: InteropInterfaceType, - Value: nil, - }, { Type: IntegerType, Value: math.Inf(1), @@ -252,6 +255,27 @@ var unmarshalJSONTestCases = []struct { }, input: `{"type":"PublicKey","value":"03b3bf1502fbdc05449b506aaf04579724024b06542e49262bfaa3f70e200040a9"}`, }, + { + input: `{"type":"InteropInterface","value":null}`, + result: Parameter{ + Type: InteropInterfaceType, + Value: nil, + }, + }, + { + input: `{"type":"InteropInterface","value":""}`, + result: Parameter{ + Type: InteropInterfaceType, + Value: nil, + }, + }, + { + input: `{"type":"InteropInterface","value":"Hundertwasser"}`, + result: Parameter{ + Type: InteropInterfaceType, + Value: nil, + }, + }, } var unmarshalJSONErrorCases = []string{ @@ -272,8 +296,6 @@ var unmarshalJSONErrorCases = []string{ `{"type": "Map","value": ["key": {}]}`, // incorrect Map value `{"type": "Map","value": ["key": {"type":"String", "value":"qwer"}, "value": {"type":"Boolean"}]}`, // incorrect Map Value value `{"type": "Map","value": ["key": {"type":"String"}, "value": {"type":"Boolean", "value":true}]}`, // incorrect Map Key value - - `{"type": "InteropInterface","value": ""}`, // ununmarshable type } func TestParam_UnmarshalJSON(t *testing.T) { From fc22a46a4c0cc6beabe88ea09806b993dc4ec873 Mon Sep 17 00:00:00 2001 From: Roman Khimov Date: Mon, 11 May 2020 01:00:19 +0300 Subject: [PATCH 09/20] rpc/server: add notification subscription Note that the protocol differs a bit from #895 in its notifications format, to avoid additional server-side processing we're omitting some metadata like: * block size and confirmations * transaction fees, confirmations, block hash and timestamp * application execution doesn't have ScriptHash populated Some block fields may also differ in encoding compared to `getblock` results (like nonce field). I think these differences are unnoticieable for most use cases, so we can leave them as is, but it can be changed in the future. --- pkg/rpc/response/events.go | 79 ++++++ pkg/rpc/response/result/application_log.go | 18 +- pkg/rpc/response/types.go | 9 + pkg/rpc/server/server.go | 316 ++++++++++++++++++++- pkg/rpc/server/server_helper_test.go | 28 +- pkg/rpc/server/subscription.go | 35 +++ pkg/rpc/server/subscription_test.go | 227 +++++++++++++++ 7 files changed, 688 insertions(+), 24 deletions(-) create mode 100644 pkg/rpc/response/events.go create mode 100644 pkg/rpc/server/subscription.go create mode 100644 pkg/rpc/server/subscription_test.go diff --git a/pkg/rpc/response/events.go b/pkg/rpc/response/events.go new file mode 100644 index 000000000..1efba39a5 --- /dev/null +++ b/pkg/rpc/response/events.go @@ -0,0 +1,79 @@ +package response + +import ( + "encoding/json" + + "github.com/pkg/errors" +) + +type ( + // EventID represents an event type happening on the chain. + EventID byte +) + +const ( + // InvalidEventID is an invalid event id that is the default value of + // EventID. It's only used as an initial value similar to nil. + InvalidEventID EventID = iota + // BlockEventID is a `block_added` event. + BlockEventID + // TransactionEventID corresponds to `transaction_added` event. + TransactionEventID + // NotificationEventID represents `notification_from_execution` events. + NotificationEventID + // ExecutionEventID is used for `transaction_executed` events. + ExecutionEventID +) + +// String is a good old Stringer implementation. +func (e EventID) String() string { + switch e { + case BlockEventID: + return "block_added" + case TransactionEventID: + return "transaction_added" + case NotificationEventID: + return "notification_from_execution" + case ExecutionEventID: + return "transaction_executed" + default: + return "unknown" + } +} + +// GetEventIDFromString converts input string into an EventID if it's possible. +func GetEventIDFromString(s string) (EventID, error) { + switch s { + case "block_added": + return BlockEventID, nil + case "transaction_added": + return TransactionEventID, nil + case "notification_from_execution": + return NotificationEventID, nil + case "transaction_executed": + return ExecutionEventID, nil + default: + return 255, errors.New("invalid stream name") + } +} + +// MarshalJSON implements json.Marshaler interface. +func (e EventID) MarshalJSON() ([]byte, error) { + return json.Marshal(e.String()) +} + +// UnmarshalJSON implements json.Unmarshaler interface. +func (e *EventID) UnmarshalJSON(b []byte) error { + var s string + + err := json.Unmarshal(b, &s) + if err != nil { + return err + } + id, err := GetEventIDFromString(s) + if err != nil { + return err + } + *e = id + return nil +} diff --git a/pkg/rpc/response/result/application_log.go b/pkg/rpc/response/result/application_log.go index da436f2f4..ee59fc43d 100644 --- a/pkg/rpc/response/result/application_log.go +++ b/pkg/rpc/response/result/application_log.go @@ -30,16 +30,22 @@ type NotificationEvent struct { Item smartcontract.Parameter `json:"state"` } +// StateEventToResultNotification converts state.NotificationEvent to +// result.NotificationEvent. +func StateEventToResultNotification(event state.NotificationEvent) NotificationEvent { + seen := make(map[vm.StackItem]bool) + item := event.Item.ToContractParameter(seen) + return NotificationEvent{ + Contract: event.ScriptHash, + Item: item, + } +} + // NewApplicationLog creates a new ApplicationLog wrapper. func NewApplicationLog(appExecRes *state.AppExecResult, scriptHash util.Uint160) ApplicationLog { events := make([]NotificationEvent, 0, len(appExecRes.Events)) for _, e := range appExecRes.Events { - seen := make(map[vm.StackItem]bool) - item := e.Item.ToContractParameter(seen) - events = append(events, NotificationEvent{ - Contract: e.ScriptHash, - Item: item, - }) + events = append(events, StateEventToResultNotification(e)) } triggerString := appExecRes.Trigger.String() diff --git a/pkg/rpc/response/types.go b/pkg/rpc/response/types.go index 0b236826a..ba23c7677 100644 --- a/pkg/rpc/response/types.go +++ b/pkg/rpc/response/types.go @@ -37,3 +37,12 @@ type GetRawTx struct { HeaderAndError Result *result.TransactionOutputRaw `json:"result"` } + +// Notification is a type used to represent wire format of events, they're +// special in that they look like requests but they don't have IDs and their +// "method" is actually an event name. +type Notification struct { + JSONRPC string `json:"jsonrpc"` + Event EventID `json:"method"` + Payload []interface{} `json:"params"` +} diff --git a/pkg/rpc/server/server.go b/pkg/rpc/server/server.go index 2d7a9e226..a7151a669 100644 --- a/pkg/rpc/server/server.go +++ b/pkg/rpc/server/server.go @@ -9,6 +9,7 @@ import ( "net" "net/http" "strconv" + "sync" "time" "github.com/gorilla/websocket" @@ -42,6 +43,19 @@ type ( coreServer *network.Server log *zap.Logger https *http.Server + shutdown chan struct{} + + subsLock sync.RWMutex + subscribers map[*subscriber]bool + subsGroup sync.WaitGroup + blockSubs int + executionSubs int + notificationSubs int + transactionSubs int + blockCh chan *block.Block + executionCh chan *state.AppExecResult + notificationCh chan *state.NotificationEvent + transactionCh chan *transaction.Transaction } ) @@ -57,6 +71,11 @@ const ( // Write deadline. wsWriteLimit = wsPingPeriod / 2 + + // Maximum number of subscribers per Server. Each websocket client is + // treated like subscriber, so technically it's a limit on websocket + // connections. + maxSubscribers = 64 ) var rpcHandlers = map[string]func(*Server, request.Params) (interface{}, *response.Error){ @@ -92,6 +111,11 @@ var rpcHandlers = map[string]func(*Server, request.Params) (interface{}, *respon "validateaddress": (*Server).validateAddress, } +var rpcWsHandlers = map[string]func(*Server, request.Params, *subscriber) (interface{}, *response.Error){ + "subscribe": (*Server).subscribe, + "unsubscribe": (*Server).unsubscribe, +} + var invalidBlockHeightError = func(index int, height int) *response.Error { return response.NewRPCError(fmt.Sprintf("Param at index %d should be greater than or equal to 0 and less then or equal to current block height, got: %d", index, height), "", nil) } @@ -120,6 +144,14 @@ func New(chain blockchainer.Blockchainer, conf rpc.Config, coreServer *network.S coreServer: coreServer, log: log, https: tlsServer, + shutdown: make(chan struct{}), + + subscribers: make(map[*subscriber]bool), + // These are NOT buffered to preserve original order of events. + blockCh: make(chan *block.Block), + executionCh: make(chan *state.AppExecResult), + notificationCh: make(chan *state.NotificationEvent), + transactionCh: make(chan *transaction.Transaction), } } @@ -134,6 +166,7 @@ func (s *Server) Start(errChan chan error) { s.Handler = http.HandlerFunc(s.handleHTTPRequest) s.log.Info("starting rpc-server", zap.String("endpoint", s.Addr)) + go s.handleSubEvents() if cfg := s.config.TLSConfig; cfg.Enabled { s.https.Handler = http.HandlerFunc(s.handleHTTPRequest) s.log.Info("starting rpc-server (https)", zap.String("endpoint", s.https.Addr)) @@ -156,6 +189,10 @@ func (s *Server) Start(errChan chan error) { // method. func (s *Server) Shutdown() error { var httpsErr error + + // Signal to websocket writer routines and handleSubEvents. + close(s.shutdown) + if s.config.TLSConfig.Enabled { s.log.Info("shutting down rpc-server (https)", zap.String("endpoint", s.https.Addr)) httpsErr = s.https.Shutdown(context.Background()) @@ -163,6 +200,10 @@ func (s *Server) Shutdown() error { s.log.Info("shutting down rpc-server", zap.String("endpoint", s.Addr)) err := s.Server.Shutdown(context.Background()) + + // Wait for handleSubEvents to finish. + <-s.executionCh + if err == nil { return httpsErr } @@ -170,20 +211,40 @@ func (s *Server) Shutdown() error { } func (s *Server) handleHTTPRequest(w http.ResponseWriter, httpRequest *http.Request) { + req := request.NewIn() + if httpRequest.URL.Path == "/ws" && httpRequest.Method == "GET" { + // Technically there is a race between this check and + // s.subscribers modification 20 lines below, but it's tiny + // and not really critical to bother with it. Some additional + // clients may sneak in, no big deal. + s.subsLock.RLock() + numOfSubs := len(s.subscribers) + s.subsLock.RUnlock() + if numOfSubs >= maxSubscribers { + s.writeHTTPErrorResponse( + req, + w, + response.NewInternalServerError("websocket users limit reached", nil), + ) + return + } ws, err := upgrader.Upgrade(w, httpRequest, nil) if err != nil { s.log.Info("websocket connection upgrade failed", zap.Error(err)) return } resChan := make(chan response.Raw) - go s.handleWsWrites(ws, resChan) - s.handleWsReads(ws, resChan) + subChan := make(chan *websocket.PreparedMessage, notificationBufSize) + subscr := &subscriber{writer: subChan, ws: ws} + s.subsLock.Lock() + s.subscribers[subscr] = true + s.subsLock.Unlock() + go s.handleWsWrites(ws, resChan, subChan) + s.handleWsReads(ws, resChan, subscr) return } - req := request.NewIn() - if httpRequest.Method != "POST" { s.writeHTTPErrorResponse( req, @@ -201,11 +262,14 @@ func (s *Server) handleHTTPRequest(w http.ResponseWriter, httpRequest *http.Requ return } - resp := s.handleRequest(req) + resp := s.handleRequest(req, nil) s.writeHTTPServerResponse(req, w, resp) } -func (s *Server) handleRequest(req *request.In) response.Raw { +func (s *Server) handleRequest(req *request.In, sub *subscriber) response.Raw { + var res interface{} + var resErr *response.Error + reqParams, err := req.Params() if err != nil { return s.packResponseToRaw(req, nil, response.NewInvalidParamsError("Problem parsing request parameters", err)) @@ -217,20 +281,37 @@ func (s *Server) handleRequest(req *request.In) response.Raw { incCounter(req.Method) + resErr = response.NewMethodNotFoundError(fmt.Sprintf("Method '%s' not supported", req.Method), nil) handler, ok := rpcHandlers[req.Method] - if !ok { - return s.packResponseToRaw(req, nil, response.NewMethodNotFoundError(fmt.Sprintf("Method '%s' not supported", req.Method), nil)) + if ok { + res, resErr = handler(s, *reqParams) + } else if sub != nil { + handler, ok := rpcWsHandlers[req.Method] + if ok { + res, resErr = handler(s, *reqParams, sub) + } } - res, resErr := handler(s, *reqParams) return s.packResponseToRaw(req, res, resErr) } -func (s *Server) handleWsWrites(ws *websocket.Conn, resChan <-chan response.Raw) { +func (s *Server) handleWsWrites(ws *websocket.Conn, resChan <-chan response.Raw, subChan <-chan *websocket.PreparedMessage) { pingTicker := time.NewTicker(wsPingPeriod) defer ws.Close() defer pingTicker.Stop() for { select { + case <-s.shutdown: + // Signal to the reader routine. + ws.Close() + return + case event, ok := <-subChan: + if !ok { + return + } + ws.SetWriteDeadline(time.Now().Add(wsWriteLimit)) + if err := ws.WritePreparedMessage(event); err != nil { + return + } case res, ok := <-resChan: if !ok { return @@ -248,22 +329,36 @@ func (s *Server) handleWsWrites(ws *websocket.Conn, resChan <-chan response.Raw) } } -func (s *Server) handleWsReads(ws *websocket.Conn, resChan chan<- response.Raw) { +func (s *Server) handleWsReads(ws *websocket.Conn, resChan chan<- response.Raw, subscr *subscriber) { ws.SetReadLimit(wsReadLimit) ws.SetReadDeadline(time.Now().Add(wsPongLimit)) ws.SetPongHandler(func(string) error { ws.SetReadDeadline(time.Now().Add(wsPongLimit)); return nil }) +requestloop: for { req := new(request.In) err := ws.ReadJSON(req) if err != nil { break } - res := s.handleRequest(req) + res := s.handleRequest(req, subscr) if res.Error != nil { s.logRequestError(req, res.Error) } - resChan <- res + select { + case <-s.shutdown: + break requestloop + case resChan <- res: + } + } + s.subsLock.Lock() + delete(s.subscribers, subscr) + for _, e := range subscr.feeds { + if e != response.InvalidEventID { + s.unsubscribeFromChannel(e) + } + } + s.subsLock.Unlock() close(resChan) ws.Close() } @@ -1024,6 +1119,201 @@ func (s *Server) sendrawtransaction(reqParams request.Params) (interface{}, *res return results, resultsErr } +// subscribe handles subscription requests from websocket clients. +func (s *Server) subscribe(reqParams request.Params, sub *subscriber) (interface{}, *response.Error) { + p, ok := reqParams.Value(0) + if !ok { + return nil, response.ErrInvalidParams + } + streamName, err := p.GetString() + if err != nil { + return nil, response.ErrInvalidParams + } + event, err := response.GetEventIDFromString(streamName) + if err != nil { + return nil, response.ErrInvalidParams + } + s.subsLock.Lock() + defer s.subsLock.Unlock() + select { + case <-s.shutdown: + return nil, response.NewInternalServerError("server is shutting down", nil) + default: + } + var id int + for ; id < len(sub.feeds); id++ { + if sub.feeds[id] == response.InvalidEventID { + break + } + } + if id == len(sub.feeds) { + return nil, response.NewInternalServerError("maximum number of subscriptions is reached", nil) + } + sub.feeds[id] = event + s.subscribeToChannel(event) + return strconv.FormatInt(int64(id), 10), nil +} + +// subscribeToChannel subscribes RPC server to appropriate chain events if +// it's not yet subscribed for them. It's supposed to be called with s.subsLock +// taken by the caller. +func (s *Server) subscribeToChannel(event response.EventID) { + switch event { + case response.BlockEventID: + if s.blockSubs == 0 { + s.chain.SubscribeForBlocks(s.blockCh) + } + s.blockSubs++ + case response.TransactionEventID: + if s.transactionSubs == 0 { + s.chain.SubscribeForTransactions(s.transactionCh) + } + s.transactionSubs++ + case response.NotificationEventID: + if s.notificationSubs == 0 { + s.chain.SubscribeForNotifications(s.notificationCh) + } + s.notificationSubs++ + case response.ExecutionEventID: + if s.executionSubs == 0 { + s.chain.SubscribeForExecutions(s.executionCh) + } + s.executionSubs++ + } +} + +// unsubscribe handles unsubscription requests from websocket clients. +func (s *Server) unsubscribe(reqParams request.Params, sub *subscriber) (interface{}, *response.Error) { + p, ok := reqParams.Value(0) + if !ok { + return nil, response.ErrInvalidParams + } + id, err := p.GetInt() + if err != nil || id < 0 { + return nil, response.ErrInvalidParams + } + s.subsLock.Lock() + defer s.subsLock.Unlock() + if len(sub.feeds) <= id || sub.feeds[id] == response.InvalidEventID { + return nil, response.ErrInvalidParams + } + event := sub.feeds[id] + sub.feeds[id] = response.InvalidEventID + s.unsubscribeFromChannel(event) + return true, nil +} + +// unsubscribeFromChannel unsubscribes RPC server from appropriate chain events +// if there are no other subscribers for it. It's supposed to be called with +// s.subsLock taken by the caller. +func (s *Server) unsubscribeFromChannel(event response.EventID) { + switch event { + case response.BlockEventID: + s.blockSubs-- + if s.blockSubs == 0 { + s.chain.UnsubscribeFromBlocks(s.blockCh) + } + case response.TransactionEventID: + s.transactionSubs-- + if s.transactionSubs == 0 { + s.chain.UnsubscribeFromTransactions(s.transactionCh) + } + case response.NotificationEventID: + s.notificationSubs-- + if s.notificationSubs == 0 { + s.chain.UnsubscribeFromNotifications(s.notificationCh) + } + case response.ExecutionEventID: + s.executionSubs-- + if s.executionSubs == 0 { + s.chain.UnsubscribeFromExecutions(s.executionCh) + } + } +} + +func (s *Server) handleSubEvents() { +chloop: + for { + var resp = response.Notification{ + JSONRPC: request.JSONRPCVersion, + Payload: make([]interface{}, 1), + } + var msg *websocket.PreparedMessage + select { + case <-s.shutdown: + break chloop + case b := <-s.blockCh: + resp.Event = response.BlockEventID + resp.Payload[0] = b + case execution := <-s.executionCh: + resp.Event = response.ExecutionEventID + resp.Payload[0] = result.NewApplicationLog(execution, util.Uint160{}) + case notification := <-s.notificationCh: + resp.Event = response.NotificationEventID + resp.Payload[0] = result.StateEventToResultNotification(*notification) + case tx := <-s.transactionCh: + resp.Event = response.TransactionEventID + resp.Payload[0] = tx + } + s.subsLock.RLock() + subloop: + for sub := range s.subscribers { + for _, subID := range sub.feeds { + if subID == resp.Event { + if msg == nil { + b, err := json.Marshal(resp) + if err != nil { + s.log.Error("failed to marshal notification", + zap.Error(err), + zap.String("type", resp.Event.String())) + break subloop + } + msg, err = websocket.NewPreparedMessage(websocket.TextMessage, b) + if err != nil { + s.log.Error("failed to prepare notification message", + zap.Error(err), + zap.String("type", resp.Event.String())) + break subloop + } + } + sub.writer <- msg + // The message is sent only once per subscriber. + break + } + } + } + s.subsLock.RUnlock() + } + // It's important to do it with lock held because no subscription routine + // should be running concurrently to this one. And even if one is to run + // after unlock, it'll see closed s.shutdown and won't subscribe. + s.subsLock.Lock() + // There might be no subscription in reality, but it's not a problem as + // core.Blockchain allows unsubscribing non-subscribed channels. + s.chain.UnsubscribeFromBlocks(s.blockCh) + s.chain.UnsubscribeFromTransactions(s.transactionCh) + s.chain.UnsubscribeFromNotifications(s.notificationCh) + s.chain.UnsubscribeFromExecutions(s.executionCh) + s.subsLock.Unlock() +drainloop: + for { + select { + case <-s.blockCh: + case <-s.executionCh: + case <-s.notificationCh: + case <-s.transactionCh: + default: + break drainloop + } + } + // It's not required closing these, but since they're drained already + // this is safe and it also allows to give a signal to Shutdown routine. + close(s.blockCh) + close(s.transactionCh) + close(s.notificationCh) + close(s.executionCh) +} + func (s *Server) blockHeightFromParam(param *request.Param) (int, *response.Error) { num, err := param.GetInt() if err != nil { diff --git a/pkg/rpc/server/server_helper_test.go b/pkg/rpc/server/server_helper_test.go index efec61f10..5e31c4ae7 100644 --- a/pkg/rpc/server/server_helper_test.go +++ b/pkg/rpc/server/server_helper_test.go @@ -14,12 +14,11 @@ import ( "github.com/nspcc-dev/neo-go/pkg/network" "github.com/nspcc-dev/neo-go/pkg/util" "github.com/stretchr/testify/require" + "go.uber.org/zap" "go.uber.org/zap/zaptest" ) -func initServerWithInMemoryChain(t *testing.T) (*core.Blockchain, *Server, *httptest.Server) { - var nBlocks uint32 - +func getUnitTestChain(t *testing.T) (*core.Blockchain, config.Config, *zap.Logger) { net := config.ModeUnitTestNet configPath := "../../../config" cfg, err := config.Load(configPath, net) @@ -32,6 +31,10 @@ func initServerWithInMemoryChain(t *testing.T) (*core.Blockchain, *Server, *http go chain.Run() + return chain, cfg, logger +} + +func getTestBlocks(t *testing.T) []*block.Block { // File "./testdata/testblocks.acc" was generated by function core._ // ("neo-go/pkg/core/helper_test.go"). // To generate new "./testdata/testblocks.acc", follow the steps: @@ -41,15 +44,21 @@ func initServerWithInMemoryChain(t *testing.T) (*core.Blockchain, *Server, *http f, err := os.Open("testdata/testblocks.acc") require.Nil(t, err) br := io.NewBinReaderFromIO(f) - nBlocks = br.ReadU32LE() + nBlocks := br.ReadU32LE() require.Nil(t, br.Err) + blocks := make([]*block.Block, 0, int(nBlocks)) for i := 0; i < int(nBlocks); i++ { _ = br.ReadU32LE() b := &block.Block{} b.DecodeBinary(br) require.Nil(t, br.Err) - require.NoError(t, chain.AddBlock(b)) + blocks = append(blocks, b) } + return blocks +} + +func initClearServerWithInMemoryChain(t *testing.T) (*core.Blockchain, *Server, *httptest.Server) { + chain, cfg, logger := getUnitTestChain(t) serverConfig := network.NewServerConfig(cfg) server, err := network.NewServer(serverConfig, chain, logger) @@ -64,6 +73,15 @@ func initServerWithInMemoryChain(t *testing.T) (*core.Blockchain, *Server, *http return chain, &rpcServer, srv } +func initServerWithInMemoryChain(t *testing.T) (*core.Blockchain, *Server, *httptest.Server) { + chain, rpcServer, srv := initClearServerWithInMemoryChain(t) + + for _, b := range getTestBlocks(t) { + require.NoError(t, chain.AddBlock(b)) + } + return chain, rpcServer, srv +} + type FeerStub struct{} func (fs *FeerStub) IsLowPriority(util.Fixed8) bool { diff --git a/pkg/rpc/server/subscription.go b/pkg/rpc/server/subscription.go new file mode 100644 index 000000000..10c9e25ec --- /dev/null +++ b/pkg/rpc/server/subscription.go @@ -0,0 +1,35 @@ +package server + +import ( + "github.com/gorilla/websocket" + "github.com/nspcc-dev/neo-go/pkg/rpc/response" +) + +type ( + // subscriber is an event subscriber. + subscriber struct { + writer chan<- *websocket.PreparedMessage + ws *websocket.Conn + + // These work like slots as there is not a lot of them (it's + // cheaper doing it this way rather than creating a map), + // pointing to EventID is an obvious overkill at the moment, but + // that's not for long. + feeds [maxFeeds]response.EventID + } +) + +const ( + // Maximum number of subscriptions per one client. + maxFeeds = 16 + + // This sets notification messages buffer depth, it may seem to be quite + // big, but there is a big gap in speed between internal event processing + // and networking communication that is combined with spiky nature of our + // event generation process, which leads to lots of events generated in + // short time and they will put some pressure to this buffer (consider + // ~500 invocation txs in one block with some notifications). At the same + // time this channel is about sending pointers, so it's doesn't cost + // a lot in terms of memory used. + notificationBufSize = 1024 +) diff --git a/pkg/rpc/server/subscription_test.go b/pkg/rpc/server/subscription_test.go new file mode 100644 index 000000000..bd4fcb792 --- /dev/null +++ b/pkg/rpc/server/subscription_test.go @@ -0,0 +1,227 @@ +package server + +import ( + "encoding/json" + "fmt" + "strings" + "testing" + "time" + + "github.com/gorilla/websocket" + "github.com/nspcc-dev/neo-go/pkg/core" + "github.com/nspcc-dev/neo-go/pkg/core/transaction" + "github.com/nspcc-dev/neo-go/pkg/rpc/response" + "github.com/stretchr/testify/require" + "go.uber.org/atomic" +) + +func wsReader(t *testing.T, ws *websocket.Conn, msgCh chan<- []byte, isFinished *atomic.Bool) { + for { + ws.SetReadDeadline(time.Now().Add(time.Second)) + _, body, err := ws.ReadMessage() + if isFinished.Load() { + require.Error(t, err) + break + } + require.NoError(t, err) + msgCh <- body + } +} + +func callWSGetRaw(t *testing.T, ws *websocket.Conn, msg string, respCh <-chan []byte) *response.Raw { + var resp = new(response.Raw) + + ws.SetWriteDeadline(time.Now().Add(time.Second)) + require.NoError(t, ws.WriteMessage(websocket.TextMessage, []byte(msg))) + + body := <-respCh + require.NoError(t, json.Unmarshal(body, resp)) + return resp +} + +func getNotification(t *testing.T, respCh <-chan []byte) *response.Notification { + var resp = new(response.Notification) + body := <-respCh + require.NoError(t, json.Unmarshal(body, resp)) + return resp +} + +func initCleanServerAndWSClient(t *testing.T) (*core.Blockchain, *Server, *websocket.Conn, chan []byte, *atomic.Bool) { + chain, rpcSrv, httpSrv := initClearServerWithInMemoryChain(t) + + dialer := websocket.Dialer{HandshakeTimeout: time.Second} + url := "ws" + strings.TrimPrefix(httpSrv.URL, "http") + "/ws" + ws, _, err := dialer.Dial(url, nil) + require.NoError(t, err) + + // Use buffered channel to read server's messages and then read expected + // responses from it. + respMsgs := make(chan []byte, 16) + finishedFlag := atomic.NewBool(false) + go wsReader(t, ws, respMsgs, finishedFlag) + return chain, rpcSrv, ws, respMsgs, finishedFlag +} + +func TestSubscriptions(t *testing.T) { + var subIDs = make([]string, 0) + var subFeeds = []string{"block_added", "transaction_added", "notification_from_execution", "transaction_executed"} + + chain, rpcSrv, c, respMsgs, finishedFlag := initCleanServerAndWSClient(t) + + defer chain.Close() + defer rpcSrv.Shutdown() + + for _, feed := range subFeeds { + var s string + resp := callWSGetRaw(t, c, fmt.Sprintf(`{ + "jsonrpc": "2.0", + "method": "subscribe", + "params": ["%s"], + "id": 1 +}`, feed), respMsgs) + require.Nil(t, resp.Error) + require.NotNil(t, resp.Result) + require.NoError(t, json.Unmarshal(resp.Result, &s)) + subIDs = append(subIDs, s) + } + + for _, b := range getTestBlocks(t) { + require.NoError(t, chain.AddBlock(b)) + for _, tx := range b.Transactions { + var mayNotify bool + + if tx.Type == transaction.InvocationType { + resp := getNotification(t, respMsgs) + require.Equal(t, response.ExecutionEventID, resp.Event) + mayNotify = true + } + for { + resp := getNotification(t, respMsgs) + if mayNotify && resp.Event == response.NotificationEventID { + continue + } + require.Equal(t, response.TransactionEventID, resp.Event) + break + } + } + resp := getNotification(t, respMsgs) + require.Equal(t, response.BlockEventID, resp.Event) + } + + for _, id := range subIDs { + var b bool + + resp := callWSGetRaw(t, c, fmt.Sprintf(`{ + "jsonrpc": "2.0", + "method": "unsubscribe", + "params": ["%s"], + "id": 1 +}`, id), respMsgs) + require.Nil(t, resp.Error) + require.NotNil(t, resp.Result) + require.NoError(t, json.Unmarshal(resp.Result, &b)) + require.Equal(t, true, b) + } + finishedFlag.CAS(false, true) + c.Close() +} + +func TestMaxSubscriptions(t *testing.T) { + var subIDs = make([]string, 0) + chain, rpcSrv, c, respMsgs, finishedFlag := initCleanServerAndWSClient(t) + + defer chain.Close() + defer rpcSrv.Shutdown() + + for i := 0; i < maxFeeds+1; i++ { + var s string + resp := callWSGetRaw(t, c, `{"jsonrpc": "2.0", "method": "subscribe", "params": ["block_added"], "id": 1}`, respMsgs) + if i < maxFeeds { + require.Nil(t, resp.Error) + require.NotNil(t, resp.Result) + require.NoError(t, json.Unmarshal(resp.Result, &s)) + // Each ID must be unique. + for _, id := range subIDs { + require.NotEqual(t, id, s) + } + subIDs = append(subIDs, s) + } else { + require.NotNil(t, resp.Error) + require.Nil(t, resp.Result) + } + } + + finishedFlag.CAS(false, true) + c.Close() +} + +func TestBadSubUnsub(t *testing.T) { + var subCases = map[string]string{ + "no params": `{"jsonrpc": "2.0", "method": "subscribe", "params": [], "id": 1}`, + "bad (non-string) event": `{"jsonrpc": "2.0", "method": "subscribe", "params": [1], "id": 1}`, + "bad (wrong) event": `{"jsonrpc": "2.0", "method": "subscribe", "params": ["block_removed"], "id": 1}`, + } + var unsubCases = map[string]string{ + "no params": `{"jsonrpc": "2.0", "method": "unsubscribe", "params": [], "id": 1}`, + "bad id": `{"jsonrpc": "2.0", "method": "unsubscribe", "params": ["vasiliy"], "id": 1}`, + "not subscribed id": `{"jsonrpc": "2.0", "method": "unsubscribe", "params": ["7"], "id": 1}`, + } + chain, rpcSrv, c, respMsgs, finishedFlag := initCleanServerAndWSClient(t) + + defer chain.Close() + defer rpcSrv.Shutdown() + + testF := func(t *testing.T, cases map[string]string) func(t *testing.T) { + return func(t *testing.T) { + for n, s := range cases { + t.Run(n, func(t *testing.T) { + resp := callWSGetRaw(t, c, s, respMsgs) + require.NotNil(t, resp.Error) + require.Nil(t, resp.Result) + }) + } + } + } + t.Run("subscribe", testF(t, subCases)) + t.Run("unsubscribe", testF(t, unsubCases)) + + finishedFlag.CAS(false, true) + c.Close() +} + +func doSomeWSRequest(t *testing.T, ws *websocket.Conn) { + ws.SetWriteDeadline(time.Now().Add(time.Second)) + // It could be just about anything including invalid request, + // we only care about server handling being active. + require.NoError(t, ws.WriteMessage(websocket.TextMessage, []byte(`{"jsonrpc": "2.0", "method": "getversion", "params": [], "id": 1}`))) + ws.SetReadDeadline(time.Now().Add(time.Second)) + _, _, err := ws.ReadMessage() + require.NoError(t, err) +} + +func TestWSClientsLimit(t *testing.T) { + chain, rpcSrv, httpSrv := initClearServerWithInMemoryChain(t) + defer chain.Close() + defer rpcSrv.Shutdown() + + dialer := websocket.Dialer{HandshakeTimeout: time.Second} + url := "ws" + strings.TrimPrefix(httpSrv.URL, "http") + "/ws" + wss := make([]*websocket.Conn, maxSubscribers) + + for i := 0; i < len(wss)+1; i++ { + ws, _, err := dialer.Dial(url, nil) + if i < maxSubscribers { + require.NoError(t, err) + wss[i] = ws + // Check that it's completely ready. + doSomeWSRequest(t, ws) + } else { + require.Error(t, err) + } + } + // Check connections are still alive (it actually is necessary to add + // some use of wss to keep connections alive). + for i := 0; i < len(wss); i++ { + doSomeWSRequest(t, wss[i]) + } +} From bef14977a28b3a608bfbf5e8eac7ab9f97849257 Mon Sep 17 00:00:00 2001 From: Roman Khimov Date: Tue, 12 May 2020 11:18:44 +0300 Subject: [PATCH 10/20] rpc/client: add notifications support for WSClient It differs from #895 design in that we have Notifications channel always exposed as WSClient field, probably it simplifies things a little. --- pkg/rpc/client/wsclient.go | 143 +++++++++++++++++++++++++++++--- pkg/rpc/client/wsclient_test.go | 130 +++++++++++++++++++++++++++++ 2 files changed, 261 insertions(+), 12 deletions(-) diff --git a/pkg/rpc/client/wsclient.go b/pkg/rpc/client/wsclient.go index bdac24816..6774dd421 100644 --- a/pkg/rpc/client/wsclient.go +++ b/pkg/rpc/client/wsclient.go @@ -7,8 +7,11 @@ import ( "time" "github.com/gorilla/websocket" + "github.com/nspcc-dev/neo-go/pkg/core/block" + "github.com/nspcc-dev/neo-go/pkg/core/transaction" "github.com/nspcc-dev/neo-go/pkg/rpc/request" "github.com/nspcc-dev/neo-go/pkg/rpc/response" + "github.com/nspcc-dev/neo-go/pkg/rpc/response/result" ) // WSClient is a websocket-enabled RPC client that can be used with appropriate @@ -17,12 +20,28 @@ import ( // that is only provided via websockets (like event subscription mechanism). type WSClient struct { Client + // Notifications is a channel that is used to send events received from + // server. Client's code is supposed to be reading from this channel if + // it wants to use subscription mechanism, failing to do so will cause + // WSClient to block even regular requests. This channel is not buffered. + // In case of protocol error or upon connection closure this channel will + // be closed, so make sure to handle this. + Notifications chan Notification + ws *websocket.Conn done chan struct{} - notifications chan *request.In responses chan *response.Raw requests chan *request.Raw shutdown chan struct{} + subscriptions map[string]bool +} + +// Notification represents server-generated notification for client subscriptions. +// Value can be one of block.Block, result.ApplicationLog, result.NotificationEvent +// or transaction.Transaction based on Type. +type Notification struct { + Type response.EventID + Value interface{} } // requestResponse is a combined type for request and response since we can get @@ -59,12 +78,15 @@ func NewWS(ctx context.Context, endpoint string, opts Options) (*WSClient, error return nil, err } wsc := &WSClient{ - Client: *cl, - ws: ws, - shutdown: make(chan struct{}), - done: make(chan struct{}), - responses: make(chan *response.Raw), - requests: make(chan *request.Raw), + Client: *cl, + Notifications: make(chan Notification), + + ws: ws, + shutdown: make(chan struct{}), + done: make(chan struct{}), + responses: make(chan *response.Raw), + requests: make(chan *request.Raw), + subscriptions: make(map[string]bool), } go wsc.wsReader() go wsc.wsWriter() @@ -86,6 +108,7 @@ func (c *WSClient) Close() { func (c *WSClient) wsReader() { c.ws.SetReadLimit(wsReadLimit) c.ws.SetPongHandler(func(string) error { c.ws.SetReadDeadline(time.Now().Add(wsPongLimit)); return nil }) +readloop: for { rr := new(requestResponse) c.ws.SetReadDeadline(time.Now().Add(wsPongLimit)) @@ -95,9 +118,37 @@ func (c *WSClient) wsReader() { break } if rr.RawID == nil && rr.Method != "" { - if c.notifications != nil { - c.notifications <- &rr.In + event, err := response.GetEventIDFromString(rr.Method) + if err != nil { + // Bad event received. + break } + var slice []json.RawMessage + err = json.Unmarshal(rr.RawParams, &slice) + if err != nil || len(slice) != 1 { + // Bad event received. + break + } + var val interface{} + switch event { + case response.BlockEventID: + val = new(block.Block) + case response.TransactionEventID: + val = new(transaction.Transaction) + case response.NotificationEventID: + val = new(result.NotificationEvent) + case response.ExecutionEventID: + val = new(result.ApplicationLog) + default: + // Bad event received. + break readloop + } + err = json.Unmarshal(slice[0], val) + if err != nil || len(slice) != 1 { + // Bad event received. + break + } + c.Notifications <- Notification{event, val} } else if rr.RawID != nil && (rr.Error != nil || rr.Result != nil) { resp := new(response.Raw) resp.ID = rr.RawID @@ -112,9 +163,7 @@ func (c *WSClient) wsReader() { } close(c.done) close(c.responses) - if c.notifications != nil { - close(c.notifications) - } + close(c.Notifications) } func (c *WSClient) wsWriter() { @@ -158,3 +207,73 @@ func (c *WSClient) makeWsRequest(r *request.Raw) (*response.Raw, error) { return resp, nil } } + +func (c *WSClient) performSubscription(params request.RawParams) (string, error) { + var resp string + + if err := c.performRequest("subscribe", params, &resp); err != nil { + return "", err + } + c.subscriptions[resp] = true + return resp, nil +} + +func (c *WSClient) performUnsubscription(id string) error { + var resp bool + + if !c.subscriptions[id] { + return errors.New("no subscription with this ID") + } + if err := c.performRequest("unsubscribe", request.NewRawParams(id), &resp); err != nil { + return err + } + if !resp { + return errors.New("unsubscribe method returned false result") + } + delete(c.subscriptions, id) + return nil +} + +// SubscribeForNewBlocks adds subscription for new block events to this instance +// of client. +func (c *WSClient) SubscribeForNewBlocks() (string, error) { + params := request.NewRawParams("block_added") + return c.performSubscription(params) +} + +// SubscribeForNewTransactions adds subscription for new transaction events to +// this instance of client. +func (c *WSClient) SubscribeForNewTransactions() (string, error) { + params := request.NewRawParams("transaction_added") + return c.performSubscription(params) +} + +// SubscribeForExecutionNotifications adds subscription for notifications +// generated during transaction execution to this instance of client. +func (c *WSClient) SubscribeForExecutionNotifications() (string, error) { + params := request.NewRawParams("notification_from_execution") + return c.performSubscription(params) +} + +// SubscribeForTransactionExecutions adds subscription for application execution +// results generated during transaction execution to this instance of client. +func (c *WSClient) SubscribeForTransactionExecutions() (string, error) { + params := request.NewRawParams("transaction_executed") + return c.performSubscription(params) +} + +// Unsubscribe removes subscription for given event stream. +func (c *WSClient) Unsubscribe(id string) error { + return c.performUnsubscription(id) +} + +// UnsubscribeAll removes all active subscriptions of current client. +func (c *WSClient) UnsubscribeAll() error { + for id := range c.subscriptions { + err := c.performUnsubscription(id) + if err != nil { + return err + } + } + return nil +} diff --git a/pkg/rpc/client/wsclient_test.go b/pkg/rpc/client/wsclient_test.go index 2a996999a..f78481f05 100644 --- a/pkg/rpc/client/wsclient_test.go +++ b/pkg/rpc/client/wsclient_test.go @@ -2,8 +2,12 @@ package client import ( "context" + "net/http" + "net/http/httptest" "testing" + "time" + "github.com/gorilla/websocket" "github.com/stretchr/testify/require" ) @@ -14,3 +18,129 @@ func TestWSClientClose(t *testing.T) { require.NoError(t, err) wsc.Close() } + +func TestWSClientSubscription(t *testing.T) { + var cases = map[string]func(*WSClient) (string, error){ + "blocks": (*WSClient).SubscribeForNewBlocks, + "transactions": (*WSClient).SubscribeForNewTransactions, + "notifications": (*WSClient).SubscribeForExecutionNotifications, + "executions": (*WSClient).SubscribeForTransactionExecutions, + } + t.Run("good", func(t *testing.T) { + for name, f := range cases { + t.Run(name, func(t *testing.T) { + srv := initTestServer(t, `{"jsonrpc": "2.0", "id": 1, "result": "55aaff00"}`) + defer srv.Close() + wsc, err := NewWS(context.TODO(), httpURLtoWS(srv.URL), Options{}) + require.NoError(t, err) + id, err := f(wsc) + require.NoError(t, err) + require.Equal(t, "55aaff00", id) + }) + } + }) + t.Run("bad", func(t *testing.T) { + for name, f := range cases { + t.Run(name, func(t *testing.T) { + srv := initTestServer(t, `{"jsonrpc": "2.0", "id": 1, "error":{"code":-32602,"message":"Invalid Params"}}`) + defer srv.Close() + wsc, err := NewWS(context.TODO(), httpURLtoWS(srv.URL), Options{}) + require.NoError(t, err) + _, err = f(wsc) + require.Error(t, err) + }) + } + }) +} + +func TestWSClientUnsubscription(t *testing.T) { + type responseCheck struct { + response string + code func(*testing.T, *WSClient) + } + var cases = map[string]responseCheck{ + "good": {`{"jsonrpc": "2.0", "id": 1, "result": true}`, func(t *testing.T, wsc *WSClient) { + // We can't really subscribe using this stub server, so set up wsc internals. + wsc.subscriptions["0"] = true + err := wsc.Unsubscribe("0") + require.NoError(t, err) + }}, + "all": {`{"jsonrpc": "2.0", "id": 1, "result": true}`, func(t *testing.T, wsc *WSClient) { + // We can't really subscribe using this stub server, so set up wsc internals. + wsc.subscriptions["0"] = true + err := wsc.UnsubscribeAll() + require.NoError(t, err) + require.Equal(t, 0, len(wsc.subscriptions)) + }}, + "not subscribed": {`{"jsonrpc": "2.0", "id": 1, "result": true}`, func(t *testing.T, wsc *WSClient) { + err := wsc.Unsubscribe("0") + require.Error(t, err) + }}, + "error returned": {`{"jsonrpc": "2.0", "id": 1, "error":{"code":-32602,"message":"Invalid Params"}}`, func(t *testing.T, wsc *WSClient) { + // We can't really subscribe using this stub server, so set up wsc internals. + wsc.subscriptions["0"] = true + err := wsc.Unsubscribe("0") + require.Error(t, err) + }}, + "false returned": {`{"jsonrpc": "2.0", "id": 1, "result": false}`, func(t *testing.T, wsc *WSClient) { + // We can't really subscribe using this stub server, so set up wsc internals. + wsc.subscriptions["0"] = true + err := wsc.Unsubscribe("0") + require.Error(t, err) + }}, + } + for name, rc := range cases { + t.Run(name, func(t *testing.T) { + srv := initTestServer(t, rc.response) + defer srv.Close() + wsc, err := NewWS(context.TODO(), httpURLtoWS(srv.URL), Options{}) + require.NoError(t, err) + rc.code(t, wsc) + }) + } +} + +func TestWSClientEvents(t *testing.T) { + var ok bool + // Events from RPC server test chain. + var events = []string{ + `{"jsonrpc":"2.0","method":"transaction_executed","params":[{"txid":"0xe1cd5e57e721d2a2e05fb1f08721b12057b25ab1dd7fd0f33ee1639932fdfad7","executions":[{"trigger":"Application","contract":"0x0000000000000000000000000000000000000000","vmstate":"HALT","gas_consumed":"2.291","stack":[],"notifications":[{"contract":"0x1b4357bff5a01bdf2a6581247cf9ed1e24629176","state":{"type":"Array","value":[{"type":"ByteArray","value":"636f6e74726163742063616c6c"},{"type":"ByteArray","value":"7472616e73666572"},{"type":"Array","value":[{"type":"ByteArray","value":"769162241eedf97c2481652adf1ba0f5bf57431b"},{"type":"ByteArray","value":"316e851039019d39dfc2c37d6c3fee19fd580987"},{"type":"Integer","value":"1000"}]}]}},{"contract":"0x1b4357bff5a01bdf2a6581247cf9ed1e24629176","state":{"type":"Array","value":[{"type":"ByteArray","value":"7472616e73666572"},{"type":"ByteArray","value":"769162241eedf97c2481652adf1ba0f5bf57431b"},{"type":"ByteArray","value":"316e851039019d39dfc2c37d6c3fee19fd580987"},{"type":"Integer","value":"1000"}]}}]}]}]}`, + `{"jsonrpc":"2.0","method":"notification_from_execution","params":[{"contract":"0x1b4357bff5a01bdf2a6581247cf9ed1e24629176","state":{"type":"Array","value":[{"type":"ByteArray","value":"636f6e74726163742063616c6c"},{"type":"ByteArray","value":"7472616e73666572"},{"type":"Array","value":[{"type":"ByteArray","value":"769162241eedf97c2481652adf1ba0f5bf57431b"},{"type":"ByteArray","value":"316e851039019d39dfc2c37d6c3fee19fd580987"},{"type":"Integer","value":"1000"}]}]}}]}`, + `{"jsonrpc":"2.0","method":"transaction_added","params":[{"txid":"0xe1cd5e57e721d2a2e05fb1f08721b12057b25ab1dd7fd0f33ee1639932fdfad7","size":277,"type":"InvocationTransaction","version":1,"nonce":9,"sender":"ALHF9wsXZVEuCGgmDA6ZNsCLtrb4A1g4yG","sys_fee":"0","net_fee":"0.0037721","valid_until_block":1200,"attributes":[],"cosigners":[{"account":"0x870958fd19ee3f6c7dc3c2df399d013910856e31","scopes":1}],"vin":[],"vout":[],"scripts":[{"invocation":"0c4027727296b84853c5d9e07fb8a40e885246ae25641383b16eefbe92027ecb1635b794aacf6bbfc3e828c73829b14791c483d19eb758b57638e3191393dbf2d288","verification":"0c2102b3622bf4017bdfe317c58aed5f4c753f206b7db896046fa7d774bbc4bf7f8dc20b410a906ad4"}],"script":"01e8030c14316e851039019d39dfc2c37d6c3fee19fd5809870c14769162241eedf97c2481652adf1ba0f5bf57431b13c00c087472616e736665720c14769162241eedf97c2481652adf1ba0f5bf57431b41627d5b5238"}]}`, + `{"jsonrpc":"2.0","method":"block_added","params":[{"version":0,"previousblockhash":"0x04f7580b111ec75f0ce68d3a9fd70a0544b4521b4a98541694d8575c548b759e","merkleroot":"0xb2c7230ebee4cb83bc03afadbba413e6bca8fcdeaf9c077bea060918da0e52a1","time":1590006200,"height":207,"next_consensus":"0x4138145d67638db97b402ee0b6751ef16253ecab","script":{"invocation":"0c4063429fca5ff75c964d9e38179c75978e33f8174d91a780c2e825265cf2447281594afdd5f3e216dcaf5ff0693aec83f415996cf224454495495f6bd0a4c5d08f0c4099680903a954278580d8533121c2cd3e53a089817b6a784901ec06178a60b5f1da6e70422bdcadc89029767e08d66ce4180b99334cb2d42f42e4216394af15920c4067d5e362189e48839a24e187c59d46f5d9db862c8a029777f1548b19632bfdc73ad373827ed02369f925e89c2303b64e6b9838dca229949b9b9d3bd4c0c3ed8f0c4021d4c00d4522805883f1db929554441bcbbee127c48f6b7feeeb69a72a78c7f0a75011663e239c0820ef903f36168f42936de10f0ef20681cb735a4b53d0390f","verification":"130c2102103a7f7dd016558597f7960d27c516a4394fd968b9e65155eb4b013e4040406e0c2102a7bc55fe8684e0119768d104ba30795bdcc86619e864add26156723ed185cd620c2102b3622bf4017bdfe317c58aed5f4c753f206b7db896046fa7d774bbc4bf7f8dc20c2103d90c07df63e690ce77912e10ab51acc944b66860237b608c4f8f8309e71ee699140b413073b3bb"},"consensus_data":{"primary":0,"nonce":1111},"tx":[{"txid":"0xf736cd91ab84062a21a09b424346b241987f6245ffe8c2b2db39d595c3c222f7","size":204,"type":"InvocationTransaction","version":1,"nonce":8,"sender":"ALHF9wsXZVEuCGgmDA6ZNsCLtrb4A1g4yG","sys_fee":"0","net_fee":"0.0030421","valid_until_block":1200,"attributes":[],"cosigners":[],"vin":[],"vout":[],"scripts":[{"invocation":"0c4016e7a112742409cdfaad89dcdbcb52c94c5c1a69dfe5d8b999649eaaa787e31ca496d1734d6ea606c749ad36e9a88892240ae59e0efa7f544e0692124898d512","verification":"0c2102b3622bf4017bdfe317c58aed5f4c753f206b7db896046fa7d774bbc4bf7f8dc20b410a906ad4"}],"script":"10c00c04696e69740c14769162241eedf97c2481652adf1ba0f5bf57431b41627d5b52"},{"txid":"0xe1cd5e57e721d2a2e05fb1f08721b12057b25ab1dd7fd0f33ee1639932fdfad7","size":277,"type":"InvocationTransaction","version":1,"nonce":9,"sender":"ALHF9wsXZVEuCGgmDA6ZNsCLtrb4A1g4yG","sys_fee":"0","net_fee":"0.0037721","valid_until_block":1200,"attributes":[],"cosigners":[{"account":"0x870958fd19ee3f6c7dc3c2df399d013910856e31","scopes":1}],"vin":[],"vout":[],"scripts":[{"invocation":"0c4027727296b84853c5d9e07fb8a40e885246ae25641383b16eefbe92027ecb1635b794aacf6bbfc3e828c73829b14791c483d19eb758b57638e3191393dbf2d288","verification":"0c2102b3622bf4017bdfe317c58aed5f4c753f206b7db896046fa7d774bbc4bf7f8dc20b410a906ad4"}],"script":"01e8030c14316e851039019d39dfc2c37d6c3fee19fd5809870c14769162241eedf97c2481652adf1ba0f5bf57431b13c00c087472616e736665720c14769162241eedf97c2481652adf1ba0f5bf57431b41627d5b5238"}]}]}`, + } + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + if req.URL.Path == "/ws" && req.Method == "GET" { + var upgrader = websocket.Upgrader{} + ws, err := upgrader.Upgrade(w, req, nil) + require.NoError(t, err) + for _, event := range events { + ws.SetWriteDeadline(time.Now().Add(2 * time.Second)) + err = ws.WriteMessage(1, []byte(event)) + if err != nil { + break + } + } + ws.Close() + return + } + })) + + wsc, err := NewWS(context.TODO(), httpURLtoWS(srv.URL), Options{}) + require.NoError(t, err) + for range events { + select { + case _, ok = <-wsc.Notifications: + case <-time.After(time.Second): + t.Fatal("timeout waiting for event") + } + require.True(t, ok) + } + select { + case _, ok = <-wsc.Notifications: + case <-time.After(time.Second): + t.Fatal("timeout waiting for event") + } + // Connection closed by server. + require.False(t, ok) +} From 44ae9086b62768081aeeaf289edb6de41b2b7ed7 Mon Sep 17 00:00:00 2001 From: Roman Khimov Date: Tue, 12 May 2020 17:45:17 +0300 Subject: [PATCH 11/20] core: improve and fix locking in storeBlock Getting batch, updating Prometheus metrics and pushing events doesn't require any locking: batch is a local cache batch that no one outside cares about, Prometheus metrics are not critical to be in perfect sync and events are asynchronous anyway. Native contracts also don't require any locks and they should be processed before dumping storage changes. --- pkg/core/blockchain.go | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/pkg/core/blockchain.go b/pkg/core/blockchain.go index 54de3f1bd..7155b7734 100644 --- a/pkg/core/blockchain.go +++ b/pkg/core/blockchain.go @@ -802,12 +802,6 @@ func (bc *Blockchain) storeBlock(block *block.Block) error { } } } - bc.lock.Lock() - defer bc.lock.Unlock() - - if bc.config.SaveStorageBatch { - bc.lastBatch = cache.DAO.GetBatch() - } for i := range bc.contracts.Contracts { systemInterop := bc.newInteropContext(trigger.Application, cache, block, nil) @@ -816,14 +810,22 @@ func (bc *Blockchain) storeBlock(block *block.Block) error { } } + if bc.config.SaveStorageBatch { + bc.lastBatch = cache.DAO.GetBatch() + } + + bc.lock.Lock() _, err := cache.Persist() if err != nil { + bc.lock.Unlock() return err } bc.topBlock.Store(block) atomic.StoreUint32(&bc.blockHeight, block.Index) - updateBlockHeightMetric(block.Index) bc.memPool.RemoveStale(bc.isTxStillRelevant, bc) + bc.lock.Unlock() + + updateBlockHeightMetric(block.Index) // Genesis block is stored when Blockchain is not yet running, so there // is no one to read this event. And it doesn't make much sense as event // anyway. From c4c080d24077d7cb3a241dfdb8bcd95c1c44e2b0 Mon Sep 17 00:00:00 2001 From: Roman Khimov Date: Tue, 12 May 2020 22:38:29 +0300 Subject: [PATCH 12/20] rpc: add subscriber queue overflow check Server-side test is added, but disabled because of its unreliability. --- pkg/rpc/client/wsclient.go | 14 ++++--- pkg/rpc/client/wsclient_test.go | 1 + pkg/rpc/response/events.go | 6 +++ pkg/rpc/server/server.go | 64 +++++++++++++++++++++++------ pkg/rpc/server/subscription.go | 7 ++-- pkg/rpc/server/subscription_test.go | 40 ++++++++++++++++++ 6 files changed, 111 insertions(+), 21 deletions(-) diff --git a/pkg/rpc/client/wsclient.go b/pkg/rpc/client/wsclient.go index 6774dd421..2c144e3d6 100644 --- a/pkg/rpc/client/wsclient.go +++ b/pkg/rpc/client/wsclient.go @@ -125,7 +125,7 @@ readloop: } var slice []json.RawMessage err = json.Unmarshal(rr.RawParams, &slice) - if err != nil || len(slice) != 1 { + if err != nil || (event != response.MissedEventID && len(slice) != 1) { // Bad event received. break } @@ -139,14 +139,18 @@ readloop: val = new(result.NotificationEvent) case response.ExecutionEventID: val = new(result.ApplicationLog) + case response.MissedEventID: + // No value. default: // Bad event received. break readloop } - err = json.Unmarshal(slice[0], val) - if err != nil || len(slice) != 1 { - // Bad event received. - break + if event != response.MissedEventID { + err = json.Unmarshal(slice[0], val) + if err != nil { + // Bad event received. + break + } } c.Notifications <- Notification{event, val} } else if rr.RawID != nil && (rr.Error != nil || rr.Result != nil) { diff --git a/pkg/rpc/client/wsclient_test.go b/pkg/rpc/client/wsclient_test.go index f78481f05..708c18d45 100644 --- a/pkg/rpc/client/wsclient_test.go +++ b/pkg/rpc/client/wsclient_test.go @@ -108,6 +108,7 @@ func TestWSClientEvents(t *testing.T) { `{"jsonrpc":"2.0","method":"notification_from_execution","params":[{"contract":"0x1b4357bff5a01bdf2a6581247cf9ed1e24629176","state":{"type":"Array","value":[{"type":"ByteArray","value":"636f6e74726163742063616c6c"},{"type":"ByteArray","value":"7472616e73666572"},{"type":"Array","value":[{"type":"ByteArray","value":"769162241eedf97c2481652adf1ba0f5bf57431b"},{"type":"ByteArray","value":"316e851039019d39dfc2c37d6c3fee19fd580987"},{"type":"Integer","value":"1000"}]}]}}]}`, `{"jsonrpc":"2.0","method":"transaction_added","params":[{"txid":"0xe1cd5e57e721d2a2e05fb1f08721b12057b25ab1dd7fd0f33ee1639932fdfad7","size":277,"type":"InvocationTransaction","version":1,"nonce":9,"sender":"ALHF9wsXZVEuCGgmDA6ZNsCLtrb4A1g4yG","sys_fee":"0","net_fee":"0.0037721","valid_until_block":1200,"attributes":[],"cosigners":[{"account":"0x870958fd19ee3f6c7dc3c2df399d013910856e31","scopes":1}],"vin":[],"vout":[],"scripts":[{"invocation":"0c4027727296b84853c5d9e07fb8a40e885246ae25641383b16eefbe92027ecb1635b794aacf6bbfc3e828c73829b14791c483d19eb758b57638e3191393dbf2d288","verification":"0c2102b3622bf4017bdfe317c58aed5f4c753f206b7db896046fa7d774bbc4bf7f8dc20b410a906ad4"}],"script":"01e8030c14316e851039019d39dfc2c37d6c3fee19fd5809870c14769162241eedf97c2481652adf1ba0f5bf57431b13c00c087472616e736665720c14769162241eedf97c2481652adf1ba0f5bf57431b41627d5b5238"}]}`, `{"jsonrpc":"2.0","method":"block_added","params":[{"version":0,"previousblockhash":"0x04f7580b111ec75f0ce68d3a9fd70a0544b4521b4a98541694d8575c548b759e","merkleroot":"0xb2c7230ebee4cb83bc03afadbba413e6bca8fcdeaf9c077bea060918da0e52a1","time":1590006200,"height":207,"next_consensus":"0x4138145d67638db97b402ee0b6751ef16253ecab","script":{"invocation":"0c4063429fca5ff75c964d9e38179c75978e33f8174d91a780c2e825265cf2447281594afdd5f3e216dcaf5ff0693aec83f415996cf224454495495f6bd0a4c5d08f0c4099680903a954278580d8533121c2cd3e53a089817b6a784901ec06178a60b5f1da6e70422bdcadc89029767e08d66ce4180b99334cb2d42f42e4216394af15920c4067d5e362189e48839a24e187c59d46f5d9db862c8a029777f1548b19632bfdc73ad373827ed02369f925e89c2303b64e6b9838dca229949b9b9d3bd4c0c3ed8f0c4021d4c00d4522805883f1db929554441bcbbee127c48f6b7feeeb69a72a78c7f0a75011663e239c0820ef903f36168f42936de10f0ef20681cb735a4b53d0390f","verification":"130c2102103a7f7dd016558597f7960d27c516a4394fd968b9e65155eb4b013e4040406e0c2102a7bc55fe8684e0119768d104ba30795bdcc86619e864add26156723ed185cd620c2102b3622bf4017bdfe317c58aed5f4c753f206b7db896046fa7d774bbc4bf7f8dc20c2103d90c07df63e690ce77912e10ab51acc944b66860237b608c4f8f8309e71ee699140b413073b3bb"},"consensus_data":{"primary":0,"nonce":1111},"tx":[{"txid":"0xf736cd91ab84062a21a09b424346b241987f6245ffe8c2b2db39d595c3c222f7","size":204,"type":"InvocationTransaction","version":1,"nonce":8,"sender":"ALHF9wsXZVEuCGgmDA6ZNsCLtrb4A1g4yG","sys_fee":"0","net_fee":"0.0030421","valid_until_block":1200,"attributes":[],"cosigners":[],"vin":[],"vout":[],"scripts":[{"invocation":"0c4016e7a112742409cdfaad89dcdbcb52c94c5c1a69dfe5d8b999649eaaa787e31ca496d1734d6ea606c749ad36e9a88892240ae59e0efa7f544e0692124898d512","verification":"0c2102b3622bf4017bdfe317c58aed5f4c753f206b7db896046fa7d774bbc4bf7f8dc20b410a906ad4"}],"script":"10c00c04696e69740c14769162241eedf97c2481652adf1ba0f5bf57431b41627d5b52"},{"txid":"0xe1cd5e57e721d2a2e05fb1f08721b12057b25ab1dd7fd0f33ee1639932fdfad7","size":277,"type":"InvocationTransaction","version":1,"nonce":9,"sender":"ALHF9wsXZVEuCGgmDA6ZNsCLtrb4A1g4yG","sys_fee":"0","net_fee":"0.0037721","valid_until_block":1200,"attributes":[],"cosigners":[{"account":"0x870958fd19ee3f6c7dc3c2df399d013910856e31","scopes":1}],"vin":[],"vout":[],"scripts":[{"invocation":"0c4027727296b84853c5d9e07fb8a40e885246ae25641383b16eefbe92027ecb1635b794aacf6bbfc3e828c73829b14791c483d19eb758b57638e3191393dbf2d288","verification":"0c2102b3622bf4017bdfe317c58aed5f4c753f206b7db896046fa7d774bbc4bf7f8dc20b410a906ad4"}],"script":"01e8030c14316e851039019d39dfc2c37d6c3fee19fd5809870c14769162241eedf97c2481652adf1ba0f5bf57431b13c00c087472616e736665720c14769162241eedf97c2481652adf1ba0f5bf57431b41627d5b5238"}]}]}`, + `{"jsonrpc":"2.0","method":"event_missed","params":[]}`, } srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { if req.URL.Path == "/ws" && req.Method == "GET" { diff --git a/pkg/rpc/response/events.go b/pkg/rpc/response/events.go index 1efba39a5..3941fcfb8 100644 --- a/pkg/rpc/response/events.go +++ b/pkg/rpc/response/events.go @@ -23,6 +23,8 @@ const ( NotificationEventID // ExecutionEventID is used for `transaction_executed` events. ExecutionEventID + // MissedEventID notifies user of missed events. + MissedEventID EventID = 255 ) // String is a good old Stringer implementation. @@ -36,6 +38,8 @@ func (e EventID) String() string { return "notification_from_execution" case ExecutionEventID: return "transaction_executed" + case MissedEventID: + return "event_missed" default: return "unknown" } @@ -52,6 +56,8 @@ func GetEventIDFromString(s string) (EventID, error) { return NotificationEventID, nil case "transaction_executed": return ExecutionEventID, nil + case "event_missed": + return MissedEventID, nil default: return 255, errors.New("invalid stream name") } diff --git a/pkg/rpc/server/server.go b/pkg/rpc/server/server.go index a7151a669..18a011f87 100644 --- a/pkg/rpc/server/server.go +++ b/pkg/rpc/server/server.go @@ -296,37 +296,49 @@ func (s *Server) handleRequest(req *request.In, sub *subscriber) response.Raw { func (s *Server) handleWsWrites(ws *websocket.Conn, resChan <-chan response.Raw, subChan <-chan *websocket.PreparedMessage) { pingTicker := time.NewTicker(wsPingPeriod) - defer ws.Close() - defer pingTicker.Stop() +eventloop: for { select { case <-s.shutdown: - // Signal to the reader routine. - ws.Close() - return + break eventloop case event, ok := <-subChan: if !ok { - return + break eventloop } ws.SetWriteDeadline(time.Now().Add(wsWriteLimit)) if err := ws.WritePreparedMessage(event); err != nil { - return + break eventloop } case res, ok := <-resChan: if !ok { - return + break eventloop } ws.SetWriteDeadline(time.Now().Add(wsWriteLimit)) if err := ws.WriteJSON(res); err != nil { - return + break eventloop } case <-pingTicker.C: ws.SetWriteDeadline(time.Now().Add(wsWriteLimit)) if err := ws.WriteMessage(websocket.PingMessage, []byte{}); err != nil { - return + break eventloop } } } + ws.Close() + pingTicker.Stop() + // Drain notification channel as there might be some goroutines blocked + // on it. +drainloop: + for { + select { + case _, ok := <-subChan: + if !ok { + break drainloop + } + default: + break drainloop + } + } } func (s *Server) handleWsReads(ws *websocket.Conn, resChan chan<- response.Raw, subscr *subscriber) { @@ -1130,7 +1142,7 @@ func (s *Server) subscribe(reqParams request.Params, sub *subscriber) (interface return nil, response.ErrInvalidParams } event, err := response.GetEventIDFromString(streamName) - if err != nil { + if err != nil || event == response.MissedEventID { return nil, response.ErrInvalidParams } s.subsLock.Lock() @@ -1232,6 +1244,20 @@ func (s *Server) unsubscribeFromChannel(event response.EventID) { } func (s *Server) handleSubEvents() { + b, err := json.Marshal(response.Notification{ + JSONRPC: request.JSONRPCVersion, + Event: response.MissedEventID, + Payload: make([]interface{}, 0), + }) + if err != nil { + s.log.Error("fatal: failed to marshal overflow event", zap.Error(err)) + return + } + overflowMsg, err := websocket.NewPreparedMessage(websocket.TextMessage, b) + if err != nil { + s.log.Error("fatal: failed to prepare overflow message", zap.Error(err)) + return + } chloop: for { var resp = response.Notification{ @@ -1258,10 +1284,13 @@ chloop: s.subsLock.RLock() subloop: for sub := range s.subscribers { + if sub.overflown.Load() { + continue + } for _, subID := range sub.feeds { if subID == resp.Event { if msg == nil { - b, err := json.Marshal(resp) + b, err = json.Marshal(resp) if err != nil { s.log.Error("failed to marshal notification", zap.Error(err), @@ -1276,7 +1305,16 @@ chloop: break subloop } } - sub.writer <- msg + select { + case sub.writer <- msg: + default: + sub.overflown.Store(true) + // MissedEvent is to be delivered eventually. + go func(sub *subscriber) { + sub.writer <- overflowMsg + sub.overflown.Store(false) + }(sub) + } // The message is sent only once per subscriber. break } diff --git a/pkg/rpc/server/subscription.go b/pkg/rpc/server/subscription.go index 10c9e25ec..f4c736b08 100644 --- a/pkg/rpc/server/subscription.go +++ b/pkg/rpc/server/subscription.go @@ -3,14 +3,15 @@ package server import ( "github.com/gorilla/websocket" "github.com/nspcc-dev/neo-go/pkg/rpc/response" + "go.uber.org/atomic" ) type ( // subscriber is an event subscriber. subscriber struct { - writer chan<- *websocket.PreparedMessage - ws *websocket.Conn - + writer chan<- *websocket.PreparedMessage + ws *websocket.Conn + overflown atomic.Bool // These work like slots as there is not a lot of them (it's // cheaper doing it this way rather than creating a map), // pointing to EventID is an obvious overkill at the moment, but diff --git a/pkg/rpc/server/subscription_test.go b/pkg/rpc/server/subscription_test.go index bd4fcb792..27a5397ef 100644 --- a/pkg/rpc/server/subscription_test.go +++ b/pkg/rpc/server/subscription_test.go @@ -160,6 +160,7 @@ func TestBadSubUnsub(t *testing.T) { "no params": `{"jsonrpc": "2.0", "method": "subscribe", "params": [], "id": 1}`, "bad (non-string) event": `{"jsonrpc": "2.0", "method": "subscribe", "params": [1], "id": 1}`, "bad (wrong) event": `{"jsonrpc": "2.0", "method": "subscribe", "params": ["block_removed"], "id": 1}`, + "missed event": `{"jsonrpc": "2.0", "method": "subscribe", "params": ["event_missed"], "id": 1}`, } var unsubCases = map[string]string{ "no params": `{"jsonrpc": "2.0", "method": "unsubscribe", "params": [], "id": 1}`, @@ -225,3 +226,42 @@ func TestWSClientsLimit(t *testing.T) { doSomeWSRequest(t, wss[i]) } } + +// The purpose of this test is to overflow buffers on server side to +// receive a 'missed' event. But it's actually hard to tell when exactly +// that's going to happen because of network-level buffering, typical +// number seen in tests is around ~3500 events, but it's not reliable enough, +// thus this test is disabled. +func testSubscriptionOverflow(t *testing.T) { + const blockCnt = notificationBufSize * 5 + var receivedMiss bool + + chain, rpcSrv, c, respMsgs, finishedFlag := initCleanServerAndWSClient(t) + + defer chain.Close() + defer rpcSrv.Shutdown() + + resp := callWSGetRaw(t, c, `{"jsonrpc": "2.0","method": "subscribe","params": ["block_added"],"id": 1}`, respMsgs) + require.Nil(t, resp.Error) + require.NotNil(t, resp.Result) + + // Push a lot of new blocks, but don't read events for them. + for i := 0; i < blockCnt; i++ { + b := newBlock(t, chain, 1) + require.NoError(t, chain.AddBlock(b)) + } + for i := 0; i < blockCnt; i++ { + resp := getNotification(t, respMsgs) + if resp.Event != response.BlockEventID { + require.Equal(t, response.MissedEventID, resp.Event) + receivedMiss = true + break + } + } + require.Equal(t, true, receivedMiss) + // `Missed` is the last event and there is nothing afterwards. + require.Equal(t, 0, len(respMsgs)) + + finishedFlag.CAS(false, true) + c.Close() +} From 725b47ddefbee10027f1b7acfadb70073fcc5382 Mon Sep 17 00:00:00 2001 From: Roman Khimov Date: Wed, 13 May 2020 13:16:42 +0300 Subject: [PATCH 13/20] rpc/client: add support for notification filters Differing a bit from #895 draft specification, we won't add `sender` or `cosigner` to `transaction_executed`. --- pkg/rpc/client/wsclient.go | 38 ++++++-- pkg/rpc/client/wsclient_test.go | 160 +++++++++++++++++++++++++++++++- pkg/rpc/request/param.go | 93 +++++++++++++------ pkg/rpc/request/param_test.go | 34 ++++++- 4 files changed, 283 insertions(+), 42 deletions(-) diff --git a/pkg/rpc/client/wsclient.go b/pkg/rpc/client/wsclient.go index 2c144e3d6..24363f1dd 100644 --- a/pkg/rpc/client/wsclient.go +++ b/pkg/rpc/client/wsclient.go @@ -12,6 +12,7 @@ import ( "github.com/nspcc-dev/neo-go/pkg/rpc/request" "github.com/nspcc-dev/neo-go/pkg/rpc/response" "github.com/nspcc-dev/neo-go/pkg/rpc/response/result" + "github.com/nspcc-dev/neo-go/pkg/util" ) // WSClient is a websocket-enabled RPC client that can be used with appropriate @@ -239,30 +240,51 @@ func (c *WSClient) performUnsubscription(id string) error { } // SubscribeForNewBlocks adds subscription for new block events to this instance -// of client. -func (c *WSClient) SubscribeForNewBlocks() (string, error) { +// of client. It can filtered by primary consensus node index, nil value doesn't +// add any filters. +func (c *WSClient) SubscribeForNewBlocks(primary *int) (string, error) { params := request.NewRawParams("block_added") + if primary != nil { + params.Values = append(params.Values, request.BlockFilter{Primary: *primary}) + } return c.performSubscription(params) } // SubscribeForNewTransactions adds subscription for new transaction events to -// this instance of client. -func (c *WSClient) SubscribeForNewTransactions() (string, error) { +// this instance of client. It can be filtered by sender and/or cosigner, nil +// value is treated as missing filter. +func (c *WSClient) SubscribeForNewTransactions(sender *util.Uint160, cosigner *util.Uint160) (string, error) { params := request.NewRawParams("transaction_added") + if sender != nil || cosigner != nil { + params.Values = append(params.Values, request.TxFilter{Sender: sender, Cosigner: cosigner}) + } return c.performSubscription(params) } // SubscribeForExecutionNotifications adds subscription for notifications -// generated during transaction execution to this instance of client. -func (c *WSClient) SubscribeForExecutionNotifications() (string, error) { +// generated during transaction execution to this instance of client. It can be +// filtered by contract's hash (that emits notifications), nil value puts no such +// restrictions. +func (c *WSClient) SubscribeForExecutionNotifications(contract *util.Uint160) (string, error) { params := request.NewRawParams("notification_from_execution") + if contract != nil { + params.Values = append(params.Values, request.NotificationFilter{Contract: *contract}) + } return c.performSubscription(params) } // SubscribeForTransactionExecutions adds subscription for application execution -// results generated during transaction execution to this instance of client. -func (c *WSClient) SubscribeForTransactionExecutions() (string, error) { +// results generated during transaction execution to this instance of client. Can +// be filtered by state (HALT/FAULT) to check for successful or failing +// transactions, nil value means no filtering. +func (c *WSClient) SubscribeForTransactionExecutions(state *string) (string, error) { params := request.NewRawParams("transaction_executed") + if state != nil { + if *state != "HALT" && *state != "FAULT" { + return "", errors.New("bad state parameter") + } + params.Values = append(params.Values, request.ExecutionFilter{State: *state}) + } return c.performSubscription(params) } diff --git a/pkg/rpc/client/wsclient_test.go b/pkg/rpc/client/wsclient_test.go index 708c18d45..494d31072 100644 --- a/pkg/rpc/client/wsclient_test.go +++ b/pkg/rpc/client/wsclient_test.go @@ -8,6 +8,8 @@ import ( "time" "github.com/gorilla/websocket" + "github.com/nspcc-dev/neo-go/pkg/rpc/request" + "github.com/nspcc-dev/neo-go/pkg/util" "github.com/stretchr/testify/require" ) @@ -21,10 +23,18 @@ func TestWSClientClose(t *testing.T) { func TestWSClientSubscription(t *testing.T) { var cases = map[string]func(*WSClient) (string, error){ - "blocks": (*WSClient).SubscribeForNewBlocks, - "transactions": (*WSClient).SubscribeForNewTransactions, - "notifications": (*WSClient).SubscribeForExecutionNotifications, - "executions": (*WSClient).SubscribeForTransactionExecutions, + "blocks": func(wsc *WSClient) (string, error) { + return wsc.SubscribeForNewBlocks(nil) + }, + "transactions": func(wsc *WSClient) (string, error) { + return wsc.SubscribeForNewTransactions(nil, nil) + }, + "notifications": func(wsc *WSClient) (string, error) { + return wsc.SubscribeForExecutionNotifications(nil) + }, + "executions": func(wsc *WSClient) (string, error) { + return wsc.SubscribeForTransactionExecutions(nil) + }, } t.Run("good", func(t *testing.T) { for name, f := range cases { @@ -145,3 +155,145 @@ func TestWSClientEvents(t *testing.T) { // Connection closed by server. require.False(t, ok) } + +func TestWSExecutionVMStateCheck(t *testing.T) { + // Will answer successfully if request slips through. + srv := initTestServer(t, `{"jsonrpc": "2.0", "id": 1, "result": "55aaff00"}`) + defer srv.Close() + wsc, err := NewWS(context.TODO(), httpURLtoWS(srv.URL), Options{}) + require.NoError(t, err) + filter := "NONE" + _, err = wsc.SubscribeForTransactionExecutions(&filter) + require.Error(t, err) + wsc.Close() +} + +func TestWSFilteredSubscriptions(t *testing.T) { + var cases = []struct { + name string + clientCode func(*testing.T, *WSClient) + serverCode func(*testing.T, *request.Params) + }{ + {"blocks", + func(t *testing.T, wsc *WSClient) { + primary := 3 + _, err := wsc.SubscribeForNewBlocks(&primary) + require.NoError(t, err) + }, + func(t *testing.T, p *request.Params) { + param, ok := p.Value(1) + require.Equal(t, true, ok) + require.Equal(t, request.BlockFilterT, param.Type) + filt, ok := param.Value.(request.BlockFilter) + require.Equal(t, true, ok) + require.Equal(t, 3, filt.Primary) + }, + }, + {"transactions sender", + func(t *testing.T, wsc *WSClient) { + sender := util.Uint160{1, 2, 3, 4, 5} + _, err := wsc.SubscribeForNewTransactions(&sender, nil) + require.NoError(t, err) + }, + func(t *testing.T, p *request.Params) { + param, ok := p.Value(1) + require.Equal(t, true, ok) + require.Equal(t, request.TxFilterT, param.Type) + filt, ok := param.Value.(request.TxFilter) + require.Equal(t, true, ok) + require.Equal(t, util.Uint160{1, 2, 3, 4, 5}, *filt.Sender) + require.Nil(t, filt.Cosigner) + }, + }, + {"transactions cosigner", + func(t *testing.T, wsc *WSClient) { + cosigner := util.Uint160{0, 42} + _, err := wsc.SubscribeForNewTransactions(nil, &cosigner) + require.NoError(t, err) + }, + func(t *testing.T, p *request.Params) { + param, ok := p.Value(1) + require.Equal(t, true, ok) + require.Equal(t, request.TxFilterT, param.Type) + filt, ok := param.Value.(request.TxFilter) + require.Equal(t, true, ok) + require.Nil(t, filt.Sender) + require.Equal(t, util.Uint160{0, 42}, *filt.Cosigner) + }, + }, + {"transactions sender and cosigner", + func(t *testing.T, wsc *WSClient) { + sender := util.Uint160{1, 2, 3, 4, 5} + cosigner := util.Uint160{0, 42} + _, err := wsc.SubscribeForNewTransactions(&sender, &cosigner) + require.NoError(t, err) + }, + func(t *testing.T, p *request.Params) { + param, ok := p.Value(1) + require.Equal(t, true, ok) + require.Equal(t, request.TxFilterT, param.Type) + filt, ok := param.Value.(request.TxFilter) + require.Equal(t, true, ok) + require.Equal(t, util.Uint160{1, 2, 3, 4, 5}, *filt.Sender) + require.Equal(t, util.Uint160{0, 42}, *filt.Cosigner) + }, + }, + {"notifications", + func(t *testing.T, wsc *WSClient) { + contract := util.Uint160{1, 2, 3, 4, 5} + _, err := wsc.SubscribeForExecutionNotifications(&contract) + require.NoError(t, err) + }, + func(t *testing.T, p *request.Params) { + param, ok := p.Value(1) + require.Equal(t, true, ok) + require.Equal(t, request.NotificationFilterT, param.Type) + filt, ok := param.Value.(request.NotificationFilter) + require.Equal(t, true, ok) + require.Equal(t, util.Uint160{1, 2, 3, 4, 5}, filt.Contract) + }, + }, + {"executions", + func(t *testing.T, wsc *WSClient) { + state := "FAULT" + _, err := wsc.SubscribeForTransactionExecutions(&state) + require.NoError(t, err) + }, + func(t *testing.T, p *request.Params) { + param, ok := p.Value(1) + require.Equal(t, true, ok) + require.Equal(t, request.ExecutionFilterT, param.Type) + filt, ok := param.Value.(request.ExecutionFilter) + require.Equal(t, true, ok) + require.Equal(t, "FAULT", filt.State) + }, + }, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + if req.URL.Path == "/ws" && req.Method == "GET" { + var upgrader = websocket.Upgrader{} + ws, err := upgrader.Upgrade(w, req, nil) + require.NoError(t, err) + ws.SetReadDeadline(time.Now().Add(2 * time.Second)) + req := request.In{} + err = ws.ReadJSON(&req) + require.NoError(t, err) + params, err := req.Params() + require.NoError(t, err) + c.serverCode(t, params) + ws.SetWriteDeadline(time.Now().Add(2 * time.Second)) + err = ws.WriteMessage(1, []byte(`{"jsonrpc": "2.0", "id": 1, "result": "0"}`)) + require.NoError(t, err) + ws.Close() + } + })) + defer srv.Close() + wsc, err := NewWS(context.TODO(), httpURLtoWS(srv.URL), Options{}) + require.NoError(t, err) + c.clientCode(t, wsc) + wsc.Close() + }) + } +} diff --git a/pkg/rpc/request/param.go b/pkg/rpc/request/param.go index 55052990e..f917a33ee 100644 --- a/pkg/rpc/request/param.go +++ b/pkg/rpc/request/param.go @@ -29,6 +29,29 @@ type ( Type smartcontract.ParamType `json:"type"` Value Param `json:"value"` } + // BlockFilter is a wrapper structure for block event filter. The only + // allowed filter is primary index. + BlockFilter struct { + Primary int `json:"primary"` + } + // TxFilter is a wrapper structure for transaction event filter. It + // allows to filter transactions by senders and cosigners. + TxFilter struct { + Sender *util.Uint160 `json:"sender,omitempty"` + Cosigner *util.Uint160 `json:"cosigner,omitempty"` + } + // NotificationFilter is a wrapper structure representing filter used for + // notifications generated during transaction execution. Notifications can + // only be filtered by contract hash. + NotificationFilter struct { + Contract util.Uint160 `json:"contract"` + } + // ExecutionFilter is a wrapper structure used for transaction execution + // events. It allows to choose failing or successful transactions based + // on their VM state. + ExecutionFilter struct { + State string `json:"state"` + } ) // These are parameter types accepted by RPC server. @@ -38,6 +61,10 @@ const ( NumberT ArrayT FuncParamT + BlockFilterT + TxFilterT + NotificationFilterT + ExecutionFilterT ) func (p Param) String() string { @@ -130,38 +157,46 @@ func (p Param) GetBytesHex() ([]byte, error) { // UnmarshalJSON implements json.Unmarshaler interface. func (p *Param) UnmarshalJSON(data []byte) error { var s string - if err := json.Unmarshal(data, &s); err == nil { - p.Type = StringT - p.Value = s - - return nil - } - var num float64 - if err := json.Unmarshal(data, &num); err == nil { - p.Type = NumberT - p.Value = int(num) - - return nil + // To unmarshal correctly we need to pass pointers into the decoder. + var attempts = [...]Param{ + {NumberT, &num}, + {StringT, &s}, + {FuncParamT, &FuncParam{}}, + {BlockFilterT, &BlockFilter{}}, + {TxFilterT, &TxFilter{}}, + {NotificationFilterT, &NotificationFilter{}}, + {ExecutionFilterT, &ExecutionFilter{}}, + {ArrayT, &[]Param{}}, } - r := bytes.NewReader(data) - jd := json.NewDecoder(r) - jd.DisallowUnknownFields() - var fp FuncParam - if err := jd.Decode(&fp); err == nil { - p.Type = FuncParamT - p.Value = fp - - return nil - } - - var ps []Param - if err := json.Unmarshal(data, &ps); err == nil { - p.Type = ArrayT - p.Value = ps - - return nil + for _, cur := range attempts { + r := bytes.NewReader(data) + jd := json.NewDecoder(r) + jd.DisallowUnknownFields() + if err := jd.Decode(cur.Value); err == nil { + p.Type = cur.Type + // But we need to store actual values, not pointers. + switch val := cur.Value.(type) { + case *float64: + p.Value = int(*val) + case *string: + p.Value = *val + case *FuncParam: + p.Value = *val + case *BlockFilter: + p.Value = *val + case *TxFilter: + p.Value = *val + case *NotificationFilter: + p.Value = *val + case *ExecutionFilter: + p.Value = *val + case *[]Param: + p.Value = *val + } + return nil + } } return errors.New("unknown type") diff --git a/pkg/rpc/request/param_test.go b/pkg/rpc/request/param_test.go index 26d29bab1..032173446 100644 --- a/pkg/rpc/request/param_test.go +++ b/pkg/rpc/request/param_test.go @@ -13,7 +13,15 @@ import ( ) func TestParam_UnmarshalJSON(t *testing.T) { - msg := `["str1", 123, ["str2", 3], [{"type": "String", "value": "jajaja"}]]` + msg := `["str1", 123, ["str2", 3], [{"type": "String", "value": "jajaja"}], + {"primary": 1}, + {"sender": "f84d6a337fbc3d3a201d41da99e86b479e7a2554"}, + {"cosigner": "f84d6a337fbc3d3a201d41da99e86b479e7a2554"}, + {"sender": "f84d6a337fbc3d3a201d41da99e86b479e7a2554", "cosigner": "f84d6a337fbc3d3a201d41da99e86b479e7a2554"}, + {"contract": "f84d6a337fbc3d3a201d41da99e86b479e7a2554"}, + {"state": "HALT"}]` + contr, err := util.Uint160DecodeStringLE("f84d6a337fbc3d3a201d41da99e86b479e7a2554") + require.NoError(t, err) expected := Params{ { Type: StringT, @@ -51,6 +59,30 @@ func TestParam_UnmarshalJSON(t *testing.T) { }, }, }, + { + Type: BlockFilterT, + Value: BlockFilter{Primary: 1}, + }, + { + Type: TxFilterT, + Value: TxFilter{Sender: &contr}, + }, + { + Type: TxFilterT, + Value: TxFilter{Cosigner: &contr}, + }, + { + Type: TxFilterT, + Value: TxFilter{Sender: &contr, Cosigner: &contr}, + }, + { + Type: NotificationFilterT, + Value: NotificationFilter{Contract: contr}, + }, + { + Type: ExecutionFilterT, + Value: ExecutionFilter{State: "HALT"}, + }, } var ps Params From 2dc16c0694b507a14ce95fccadf68697bf354615 Mon Sep 17 00:00:00 2001 From: Roman Khimov Date: Thu, 14 May 2020 00:17:39 +0300 Subject: [PATCH 14/20] transaction: add json.Unmarshaler to Attribute It actually was missing and it might affect Transaction conversion to/from JSON. --- pkg/core/transaction/attribute.go | 113 ++++++++++++++++++++++++++++-- 1 file changed, 108 insertions(+), 5 deletions(-) diff --git a/pkg/core/transaction/attribute.go b/pkg/core/transaction/attribute.go index 66fe97cb4..eb75fcf8f 100644 --- a/pkg/core/transaction/attribute.go +++ b/pkg/core/transaction/attribute.go @@ -3,6 +3,7 @@ package transaction import ( "encoding/hex" "encoding/json" + "errors" "fmt" "github.com/nspcc-dev/neo-go/pkg/io" @@ -10,8 +11,14 @@ import ( // Attribute represents a Transaction attribute. type Attribute struct { - Usage AttrUsage `json:"usage"` - Data []byte `json:"data"` + Usage AttrUsage + Data []byte +} + +// attrJSON is used for JSON I/O of Attribute. +type attrJSON struct { + Usage string `json:"usage"` + Data string `json:"data"` } // DecodeBinary implements Serializable interface. @@ -70,8 +77,104 @@ func (attr *Attribute) EncodeBinary(bw *io.BinWriter) { // MarshalJSON implements the json Marshaller interface. func (attr *Attribute) MarshalJSON() ([]byte, error) { - return json.Marshal(map[string]string{ - "usage": attr.Usage.String(), - "data": hex.EncodeToString(attr.Data), + return json.Marshal(attrJSON{ + Usage: attr.Usage.String(), + Data: hex.EncodeToString(attr.Data), }) } + +// UnmarshalJSON implements the json.Unmarshaller interface. +func (attr *Attribute) UnmarshalJSON(data []byte) error { + aj := new(attrJSON) + err := json.Unmarshal(data, aj) + if err != nil { + return err + } + binData, err := hex.DecodeString(aj.Data) + if err != nil { + return err + } + switch aj.Usage { + case "ContractHash": + attr.Usage = ContractHash + case "ECDH02": + attr.Usage = ECDH02 + case "ECDH03": + attr.Usage = ECDH03 + case "Vote": + attr.Usage = Vote + case "CertURL": + attr.Usage = CertURL + case "DescriptionURL": + attr.Usage = DescriptionURL + case "Description": + attr.Usage = Description + case "Hash1": + attr.Usage = Hash1 + case "Hash2": + attr.Usage = Hash2 + case "Hash3": + attr.Usage = Hash3 + case "Hash4": + attr.Usage = Hash4 + case "Hash5": + attr.Usage = Hash5 + case "Hash6": + attr.Usage = Hash6 + case "Hash7": + attr.Usage = Hash7 + case "Hash8": + attr.Usage = Hash8 + case "Hash9": + attr.Usage = Hash9 + case "Hash10": + attr.Usage = Hash10 + case "Hash11": + attr.Usage = Hash11 + case "Hash12": + attr.Usage = Hash12 + case "Hash13": + attr.Usage = Hash13 + case "Hash14": + attr.Usage = Hash14 + case "Hash15": + attr.Usage = Hash15 + case "Remark": + attr.Usage = Remark + case "Remark1": + attr.Usage = Remark1 + case "Remark2": + attr.Usage = Remark2 + case "Remark3": + attr.Usage = Remark3 + case "Remark4": + attr.Usage = Remark4 + case "Remark5": + attr.Usage = Remark5 + case "Remark6": + attr.Usage = Remark6 + case "Remark7": + attr.Usage = Remark7 + case "Remark8": + attr.Usage = Remark8 + case "Remark9": + attr.Usage = Remark9 + case "Remark10": + attr.Usage = Remark10 + case "Remark11": + attr.Usage = Remark11 + case "Remark12": + attr.Usage = Remark12 + case "Remark13": + attr.Usage = Remark13 + case "Remark14": + attr.Usage = Remark14 + case "Remark15": + attr.Usage = Remark15 + default: + return errors.New("wrong Usage") + + } + attr.Data = binData + return nil +} From 393ce1c2302703d6b571c8c7d00494ae11c87186 Mon Sep 17 00:00:00 2001 From: Roman Khimov Date: Wed, 13 May 2020 17:13:33 +0300 Subject: [PATCH 15/20] rpc/server: add notification filters And check state string correctness on unmarshaling. --- pkg/rpc/request/param.go | 6 +- pkg/rpc/server/server.go | 45 ++++-- pkg/rpc/server/server_test.go | 10 +- pkg/rpc/server/subscription.go | 49 ++++++- pkg/rpc/server/subscription_test.go | 213 +++++++++++++++++++++++++--- 5 files changed, 284 insertions(+), 39 deletions(-) diff --git a/pkg/rpc/request/param.go b/pkg/rpc/request/param.go index f917a33ee..3f5f9acfa 100644 --- a/pkg/rpc/request/param.go +++ b/pkg/rpc/request/param.go @@ -191,7 +191,11 @@ func (p *Param) UnmarshalJSON(data []byte) error { case *NotificationFilter: p.Value = *val case *ExecutionFilter: - p.Value = *val + if (*val).State == "HALT" || (*val).State == "FAULT" { + p.Value = *val + } else { + continue + } case *[]Param: p.Value = *val } diff --git a/pkg/rpc/server/server.go b/pkg/rpc/server/server.go index 18a011f87..fad04857e 100644 --- a/pkg/rpc/server/server.go +++ b/pkg/rpc/server/server.go @@ -366,8 +366,8 @@ requestloop: s.subsLock.Lock() delete(s.subscribers, subscr) for _, e := range subscr.feeds { - if e != response.InvalidEventID { - s.unsubscribeFromChannel(e) + if e.event != response.InvalidEventID { + s.unsubscribeFromChannel(e.event) } } s.subsLock.Unlock() @@ -1145,6 +1145,31 @@ func (s *Server) subscribe(reqParams request.Params, sub *subscriber) (interface if err != nil || event == response.MissedEventID { return nil, response.ErrInvalidParams } + // Optional filter. + var filter interface{} + p, ok = reqParams.Value(1) + if ok { + switch event { + case response.BlockEventID: + if p.Type != request.BlockFilterT { + return nil, response.ErrInvalidParams + } + case response.TransactionEventID: + if p.Type != request.TxFilterT { + return nil, response.ErrInvalidParams + } + case response.NotificationEventID: + if p.Type != request.NotificationFilterT { + return nil, response.ErrInvalidParams + } + case response.ExecutionEventID: + if p.Type != request.ExecutionFilterT { + return nil, response.ErrInvalidParams + } + } + filter = p.Value + } + s.subsLock.Lock() defer s.subsLock.Unlock() select { @@ -1154,14 +1179,15 @@ func (s *Server) subscribe(reqParams request.Params, sub *subscriber) (interface } var id int for ; id < len(sub.feeds); id++ { - if sub.feeds[id] == response.InvalidEventID { + if sub.feeds[id].event == response.InvalidEventID { break } } if id == len(sub.feeds) { return nil, response.NewInternalServerError("maximum number of subscriptions is reached", nil) } - sub.feeds[id] = event + sub.feeds[id].event = event + sub.feeds[id].filter = filter s.subscribeToChannel(event) return strconv.FormatInt(int64(id), 10), nil } @@ -1206,11 +1232,12 @@ func (s *Server) unsubscribe(reqParams request.Params, sub *subscriber) (interfa } s.subsLock.Lock() defer s.subsLock.Unlock() - if len(sub.feeds) <= id || sub.feeds[id] == response.InvalidEventID { + if len(sub.feeds) <= id || sub.feeds[id].event == response.InvalidEventID { return nil, response.ErrInvalidParams } - event := sub.feeds[id] - sub.feeds[id] = response.InvalidEventID + event := sub.feeds[id].event + sub.feeds[id].event = response.InvalidEventID + sub.feeds[id].filter = nil s.unsubscribeFromChannel(event) return true, nil } @@ -1287,8 +1314,8 @@ chloop: if sub.overflown.Load() { continue } - for _, subID := range sub.feeds { - if subID == resp.Event { + for i := range sub.feeds { + if sub.feeds[i].Matches(&resp) { if msg == nil { b, err = json.Marshal(resp) if err != nil { diff --git a/pkg/rpc/server/server_test.go b/pkg/rpc/server/server_test.go index b3f64686f..f36d6c651 100644 --- a/pkg/rpc/server/server_test.go +++ b/pkg/rpc/server/server_test.go @@ -910,7 +910,7 @@ func testRPCProtocol(t *testing.T, doRPCCall func(string, string, *testing.T) [] t.Run("submit", func(t *testing.T) { rpc := `{"jsonrpc": "2.0", "id": 1, "method": "submitblock", "params": ["%s"]}` t.Run("invalid signature", func(t *testing.T) { - s := newBlock(t, chain, 1) + s := newBlock(t, chain, 1, 0) s.Script.VerificationScript[8] ^= 0xff body := doRPCCall(fmt.Sprintf(rpc, encodeBlock(t, s)), httpSrv.URL, t) checkErrGetResult(t, body, true) @@ -940,13 +940,13 @@ func testRPCProtocol(t *testing.T, doRPCCall func(string, string, *testing.T) [] } t.Run("invalid height", func(t *testing.T) { - b := newBlock(t, chain, 2, newTx()) + b := newBlock(t, chain, 2, 0, newTx()) body := doRPCCall(fmt.Sprintf(rpc, encodeBlock(t, b)), httpSrv.URL, t) checkErrGetResult(t, body, true) }) t.Run("positive", func(t *testing.T) { - b := newBlock(t, chain, 1, newTx()) + b := newBlock(t, chain, 1, 0, newTx()) body := doRPCCall(fmt.Sprintf(rpc, encodeBlock(t, b)), httpSrv.URL, t) data := checkErrGetResult(t, body, false) var res bool @@ -1114,7 +1114,7 @@ func encodeBlock(t *testing.T, b *block.Block) string { return hex.EncodeToString(w.Bytes()) } -func newBlock(t *testing.T, bc blockchainer.Blockchainer, index uint32, txs ...*transaction.Transaction) *block.Block { +func newBlock(t *testing.T, bc blockchainer.Blockchainer, index uint32, primary uint32, txs ...*transaction.Transaction) *block.Block { witness := transaction.Witness{VerificationScript: testchain.MultisigVerificationScript()} height := bc.BlockHeight() h := bc.GetHeaderHash(int(height)) @@ -1129,7 +1129,7 @@ func newBlock(t *testing.T, bc blockchainer.Blockchainer, index uint32, txs ...* Script: witness, }, ConsensusData: block.ConsensusData{ - PrimaryIndex: 0, + PrimaryIndex: primary, Nonce: 1111, }, Transactions: txs, diff --git a/pkg/rpc/server/subscription.go b/pkg/rpc/server/subscription.go index f4c736b08..16433ce51 100644 --- a/pkg/rpc/server/subscription.go +++ b/pkg/rpc/server/subscription.go @@ -2,7 +2,11 @@ package server import ( "github.com/gorilla/websocket" + "github.com/nspcc-dev/neo-go/pkg/core/block" + "github.com/nspcc-dev/neo-go/pkg/core/transaction" + "github.com/nspcc-dev/neo-go/pkg/rpc/request" "github.com/nspcc-dev/neo-go/pkg/rpc/response" + "github.com/nspcc-dev/neo-go/pkg/rpc/response/result" "go.uber.org/atomic" ) @@ -16,7 +20,11 @@ type ( // cheaper doing it this way rather than creating a map), // pointing to EventID is an obvious overkill at the moment, but // that's not for long. - feeds [maxFeeds]response.EventID + feeds [maxFeeds]feed + } + feed struct { + event response.EventID + filter interface{} } ) @@ -34,3 +42,42 @@ const ( // a lot in terms of memory used. notificationBufSize = 1024 ) + +func (f *feed) Matches(r *response.Notification) bool { + if r.Event != f.event { + return false + } + if f.filter == nil { + return true + } + switch f.event { + case response.BlockEventID: + filt := f.filter.(request.BlockFilter) + b := r.Payload[0].(*block.Block) + return int(b.ConsensusData.PrimaryIndex) == filt.Primary + case response.TransactionEventID: + filt := f.filter.(request.TxFilter) + tx := r.Payload[0].(*transaction.Transaction) + senderOK := filt.Sender == nil || tx.Sender.Equals(*filt.Sender) + cosignerOK := true + if filt.Cosigner != nil { + cosignerOK = false + for i := range tx.Cosigners { + if tx.Cosigners[i].Account.Equals(*filt.Cosigner) { + cosignerOK = true + break + } + } + } + return senderOK && cosignerOK + case response.NotificationEventID: + filt := f.filter.(request.NotificationFilter) + notification := r.Payload[0].(result.NotificationEvent) + return notification.Contract.Equals(filt.Contract) + case response.ExecutionEventID: + filt := f.filter.(request.ExecutionFilter) + applog := r.Payload[0].(result.ApplicationLog) + return len(applog.Executions) != 0 && applog.Executions[0].VMState == filt.State + } + return false +} diff --git a/pkg/rpc/server/subscription_test.go b/pkg/rpc/server/subscription_test.go index 27a5397ef..162674549 100644 --- a/pkg/rpc/server/subscription_test.go +++ b/pkg/rpc/server/subscription_test.go @@ -10,6 +10,8 @@ import ( "github.com/gorilla/websocket" "github.com/nspcc-dev/neo-go/pkg/core" "github.com/nspcc-dev/neo-go/pkg/core/transaction" + "github.com/nspcc-dev/neo-go/pkg/encoding/address" + "github.com/nspcc-dev/neo-go/pkg/internal/testchain" "github.com/nspcc-dev/neo-go/pkg/rpc/response" "github.com/stretchr/testify/require" "go.uber.org/atomic" @@ -62,6 +64,24 @@ func initCleanServerAndWSClient(t *testing.T) (*core.Blockchain, *Server, *webso return chain, rpcSrv, ws, respMsgs, finishedFlag } +func callSubscribe(t *testing.T, ws *websocket.Conn, msgs <-chan []byte, params string) string { + var s string + resp := callWSGetRaw(t, ws, fmt.Sprintf(`{"jsonrpc": "2.0","method": "subscribe","params": %s,"id": 1}`, params), msgs) + require.Nil(t, resp.Error) + require.NotNil(t, resp.Result) + require.NoError(t, json.Unmarshal(resp.Result, &s)) + return s +} + +func callUnsubscribe(t *testing.T, ws *websocket.Conn, msgs <-chan []byte, id string) { + var b bool + resp := callWSGetRaw(t, ws, fmt.Sprintf(`{"jsonrpc": "2.0","method": "unsubscribe","params": ["%s"],"id": 1}`, id), msgs) + require.Nil(t, resp.Error) + require.NotNil(t, resp.Result) + require.NoError(t, json.Unmarshal(resp.Result, &b)) + require.Equal(t, true, b) +} + func TestSubscriptions(t *testing.T) { var subIDs = make([]string, 0) var subFeeds = []string{"block_added", "transaction_added", "notification_from_execution", "transaction_executed"} @@ -72,16 +92,7 @@ func TestSubscriptions(t *testing.T) { defer rpcSrv.Shutdown() for _, feed := range subFeeds { - var s string - resp := callWSGetRaw(t, c, fmt.Sprintf(`{ - "jsonrpc": "2.0", - "method": "subscribe", - "params": ["%s"], - "id": 1 -}`, feed), respMsgs) - require.Nil(t, resp.Error) - require.NotNil(t, resp.Result) - require.NoError(t, json.Unmarshal(resp.Result, &s)) + s := callSubscribe(t, c, respMsgs, fmt.Sprintf(`["%s"]`, feed)) subIDs = append(subIDs, s) } @@ -109,23 +120,173 @@ func TestSubscriptions(t *testing.T) { } for _, id := range subIDs { - var b bool - - resp := callWSGetRaw(t, c, fmt.Sprintf(`{ - "jsonrpc": "2.0", - "method": "unsubscribe", - "params": ["%s"], - "id": 1 -}`, id), respMsgs) - require.Nil(t, resp.Error) - require.NotNil(t, resp.Result) - require.NoError(t, json.Unmarshal(resp.Result, &b)) - require.Equal(t, true, b) + callUnsubscribe(t, c, respMsgs, id) } finishedFlag.CAS(false, true) c.Close() } +func TestFilteredSubscriptions(t *testing.T) { + priv0 := testchain.PrivateKeyByID(0) + var goodSender = priv0.GetScriptHash() + + var cases = map[string]struct { + params string + check func(*testing.T, *response.Notification) + }{ + "tx matching sender": { + params: `["transaction_added", {"sender":"` + goodSender.StringLE() + `"}]`, + check: func(t *testing.T, resp *response.Notification) { + rmap := resp.Payload[0].(map[string]interface{}) + require.Equal(t, response.TransactionEventID, resp.Event) + sender := rmap["sender"].(string) + require.Equal(t, address.Uint160ToString(goodSender), sender) + }, + }, + "tx matching cosigner": { + params: `["transaction_added", {"cosigner":"` + goodSender.StringLE() + `"}]`, + check: func(t *testing.T, resp *response.Notification) { + rmap := resp.Payload[0].(map[string]interface{}) + require.Equal(t, response.TransactionEventID, resp.Event) + cosigners := rmap["cosigners"].([]interface{}) + cosigner0 := cosigners[0].(map[string]interface{}) + cosigner0acc := cosigner0["account"].(string) + require.Equal(t, "0x"+goodSender.StringLE(), cosigner0acc) + }, + }, + "tx matching sender and cosigner": { + params: `["transaction_added", {"sender":"` + goodSender.StringLE() + `", "cosigner":"` + goodSender.StringLE() + `"}]`, + check: func(t *testing.T, resp *response.Notification) { + rmap := resp.Payload[0].(map[string]interface{}) + require.Equal(t, response.TransactionEventID, resp.Event) + sender := rmap["sender"].(string) + require.Equal(t, address.Uint160ToString(goodSender), sender) + cosigners := rmap["cosigners"].([]interface{}) + cosigner0 := cosigners[0].(map[string]interface{}) + cosigner0acc := cosigner0["account"].(string) + require.Equal(t, "0x"+goodSender.StringLE(), cosigner0acc) + }, + }, + "notification matching": { + params: `["notification_from_execution", {"contract":"` + testContractHash + `"}]`, + check: func(t *testing.T, resp *response.Notification) { + rmap := resp.Payload[0].(map[string]interface{}) + require.Equal(t, response.NotificationEventID, resp.Event) + c := rmap["contract"].(string) + require.Equal(t, "0x"+testContractHash, c) + }, + }, + "execution matching": { + params: `["transaction_executed", {"state":"HALT"}]`, + check: func(t *testing.T, resp *response.Notification) { + rmap := resp.Payload[0].(map[string]interface{}) + require.Equal(t, response.ExecutionEventID, resp.Event) + execs := rmap["executions"].([]interface{}) + exec0 := execs[0].(map[string]interface{}) + st := exec0["vmstate"].(string) + require.Equal(t, "HALT", st) + }, + }, + "tx non-matching": { + params: `["transaction_added", {"sender":"00112233445566778899aabbccddeeff00112233"}]`, + check: func(t *testing.T, _ *response.Notification) { + t.Fatal("unexpected match for EnrollmentTransaction") + }, + }, + "notification non-matching": { + params: `["notification_from_execution", {"contract":"00112233445566778899aabbccddeeff00112233"}]`, + check: func(t *testing.T, _ *response.Notification) { + t.Fatal("unexpected match for contract 00112233445566778899aabbccddeeff00112233") + }, + }, + "execution non-matching": { + params: `["transaction_executed", {"state":"FAULT"}]`, + check: func(t *testing.T, _ *response.Notification) { + t.Fatal("unexpected match for faulted execution") + }, + }, + } + + for name, this := range cases { + t.Run(name, func(t *testing.T) { + chain, rpcSrv, c, respMsgs, finishedFlag := initCleanServerAndWSClient(t) + + defer chain.Close() + defer rpcSrv.Shutdown() + + // It's used as an end-of-event-stream, so it's always present. + blockSubID := callSubscribe(t, c, respMsgs, `["block_added"]`) + subID := callSubscribe(t, c, respMsgs, this.params) + + var lastBlock uint32 + for _, b := range getTestBlocks(t) { + require.NoError(t, chain.AddBlock(b)) + lastBlock = b.Index + } + + for { + resp := getNotification(t, respMsgs) + rmap := resp.Payload[0].(map[string]interface{}) + if resp.Event == response.BlockEventID { + index := rmap["height"].(float64) + if uint32(index) == lastBlock { + break + } + continue + } + this.check(t, resp) + } + + callUnsubscribe(t, c, respMsgs, subID) + callUnsubscribe(t, c, respMsgs, blockSubID) + finishedFlag.CAS(false, true) + c.Close() + }) + } +} + +func TestFilteredBlockSubscriptions(t *testing.T) { + // We can't fit this into TestFilteredSubscriptions, because it uses + // blocks as EOF events to wait for. + const numBlocks = 10 + chain, rpcSrv, c, respMsgs, finishedFlag := initCleanServerAndWSClient(t) + + defer chain.Close() + defer rpcSrv.Shutdown() + + blockSubID := callSubscribe(t, c, respMsgs, `["block_added", {"primary":3}]`) + + var expectedCnt int + for i := 0; i < numBlocks; i++ { + primary := uint32(i % 4) + if primary == 3 { + expectedCnt++ + } + b := newBlock(t, chain, 1, primary) + require.NoError(t, chain.AddBlock(b)) + } + + for i := 0; i < expectedCnt; i++ { + var resp = new(response.Notification) + select { + case body := <-respMsgs: + require.NoError(t, json.Unmarshal(body, resp)) + case <-time.After(time.Second): + t.Fatal("timeout waiting for event") + } + + require.Equal(t, response.BlockEventID, resp.Event) + rmap := resp.Payload[0].(map[string]interface{}) + cd := rmap["consensus_data"].(map[string]interface{}) + primary := cd["primary"].(float64) + require.Equal(t, 3, int(primary)) + + } + callUnsubscribe(t, c, respMsgs, blockSubID) + finishedFlag.CAS(false, true) + c.Close() +} + func TestMaxSubscriptions(t *testing.T) { var subIDs = make([]string, 0) chain, rpcSrv, c, respMsgs, finishedFlag := initCleanServerAndWSClient(t) @@ -161,6 +322,12 @@ func TestBadSubUnsub(t *testing.T) { "bad (non-string) event": `{"jsonrpc": "2.0", "method": "subscribe", "params": [1], "id": 1}`, "bad (wrong) event": `{"jsonrpc": "2.0", "method": "subscribe", "params": ["block_removed"], "id": 1}`, "missed event": `{"jsonrpc": "2.0", "method": "subscribe", "params": ["event_missed"], "id": 1}`, + "block invalid filter": `{"jsonrpc": "2.0", "method": "subscribe", "params": ["block_added", 1], "id": 1}`, + "tx filter 1": `{"jsonrpc": "2.0", "method": "subscribe", "params": ["transaction_added", 1], "id": 1}`, + "tx filter 2": `{"jsonrpc": "2.0", "method": "subscribe", "params": ["transaction_added", {"state": "HALT"}], "id": 1}`, + "notification filter": `{"jsonrpc": "2.0", "method": "subscribe", "params": ["notification_from_execution", "contract"], "id": 1}`, + "execution filter 1": `{"jsonrpc": "2.0", "method": "subscribe", "params": ["transaction_executed", "FAULT"], "id": 1}`, + "execution filter 2": `{"jsonrpc": "2.0", "method": "subscribe", "params": ["transaction_executed", {"state": "STOP"}], "id": 1}`, } var unsubCases = map[string]string{ "no params": `{"jsonrpc": "2.0", "method": "unsubscribe", "params": [], "id": 1}`, @@ -247,7 +414,7 @@ func testSubscriptionOverflow(t *testing.T) { // Push a lot of new blocks, but don't read events for them. for i := 0; i < blockCnt; i++ { - b := newBlock(t, chain, 1) + b := newBlock(t, chain, 1, 0) require.NoError(t, chain.AddBlock(b)) } for i := 0; i < blockCnt; i++ { From 7633439845173b1bd30e92959660df49623bd8da Mon Sep 17 00:00:00 2001 From: Roman Khimov Date: Wed, 13 May 2020 21:27:08 +0300 Subject: [PATCH 16/20] rpc/block: rework the way Block is JSONized Our block.Block was JSONized in a bit different fashion than result.Block in its NextConsensus and Index fields. It's not good for notifications because third-party clients would probably expect to see the same format. Also, using completely different Block representation is probably making our client a bit weaker as this representation is harder to use with other neo-go components. So use the same approach we took for Transactions and wrap block.Block which is to be serialized in proper way. Fix `Script` JSONization along the way, 3.0 node wraps it within `witnesses`. --- pkg/core/block/block.go | 54 +++++++++++- pkg/core/block/block_base.go | 76 +++++++++++++++-- pkg/core/block/block_test.go | 2 + pkg/core/block/consensus_data.go | 38 ++++++++- pkg/rpc/client/rpc_test.go | 123 +++++++++++++++++----------- pkg/rpc/client/wsclient_test.go | 2 +- pkg/rpc/response/result/block.go | 96 +++++++++++++--------- pkg/rpc/server/server_test.go | 5 +- pkg/rpc/server/subscription_test.go | 2 +- 9 files changed, 296 insertions(+), 102 deletions(-) diff --git a/pkg/core/block/block.go b/pkg/core/block/block.go index a9715d71e..14c39ce51 100644 --- a/pkg/core/block/block.go +++ b/pkg/core/block/block.go @@ -1,6 +1,7 @@ package block import ( + "encoding/json" "errors" "github.com/Workiva/go-datastructures/queue" @@ -19,10 +20,16 @@ type Block struct { ConsensusData ConsensusData `json:"consensus_data"` // Transaction list. - Transactions []*transaction.Transaction `json:"tx"` + Transactions []*transaction.Transaction // True if this block is created from trimmed data. - Trimmed bool `json:"-"` + Trimmed bool +} + +// auxBlock is used for JSON i/o. +type auxBlock struct { + ConsensusData ConsensusData `json:"consensus_data"` + Transactions []*transaction.Transaction `json:"tx"` } // Header returns the Header of the Block. @@ -179,3 +186,46 @@ func (b *Block) Compare(item queue.Item) int { return -1 } } + +// MarshalJSON implements json.Marshaler interface. +func (b Block) MarshalJSON() ([]byte, error) { + auxb, err := json.Marshal(auxBlock{ + ConsensusData: b.ConsensusData, + Transactions: b.Transactions, + }) + if err != nil { + return nil, err + } + baseBytes, err := json.Marshal(b.Base) + if err != nil { + return nil, err + } + + // Stitch them together. + if baseBytes[len(baseBytes)-1] != '}' || auxb[0] != '{' { + return nil, errors.New("can't merge internal jsons") + } + baseBytes[len(baseBytes)-1] = ',' + baseBytes = append(baseBytes, auxb[1:]...) + return baseBytes, nil +} + +// UnmarshalJSON implements json.Unmarshaler interface. +func (b *Block) UnmarshalJSON(data []byte) error { + // As Base and auxb are at the same level in json, + // do unmarshalling separately for both structs. + auxb := new(auxBlock) + err := json.Unmarshal(data, auxb) + if err != nil { + return err + } + base := new(Base) + err = json.Unmarshal(data, base) + if err != nil { + return err + } + b.Base = *base + b.Transactions = auxb.Transactions + b.ConsensusData = auxb.ConsensusData + return nil +} diff --git a/pkg/core/block/block_base.go b/pkg/core/block/block_base.go index 3b4073a5a..0a060d597 100644 --- a/pkg/core/block/block_base.go +++ b/pkg/core/block/block_base.go @@ -1,10 +1,13 @@ package block import ( + "encoding/json" + "errors" "fmt" "github.com/nspcc-dev/neo-go/pkg/core/transaction" "github.com/nspcc-dev/neo-go/pkg/crypto/hash" + "github.com/nspcc-dev/neo-go/pkg/encoding/address" "github.com/nspcc-dev/neo-go/pkg/io" "github.com/nspcc-dev/neo-go/pkg/util" ) @@ -12,31 +15,31 @@ import ( // Base holds the base info of a block type Base struct { // Version of the block. - Version uint32 `json:"version"` + Version uint32 // hash of the previous block. - PrevHash util.Uint256 `json:"previousblockhash"` + PrevHash util.Uint256 // Root hash of a transaction list. - MerkleRoot util.Uint256 `json:"merkleroot"` + MerkleRoot util.Uint256 // Timestamp is a millisecond-precision timestamp. // The time stamp of each block must be later than previous block's time stamp. // Generally the difference of two block's time stamp is about 15 seconds and imprecision is allowed. // The height of the block must be exactly equal to the height of the previous block plus 1. - Timestamp uint64 `json:"time"` + Timestamp uint64 // index/height of the block - Index uint32 `json:"height"` + Index uint32 // Contract address of the next miner - NextConsensus util.Uint160 `json:"next_consensus"` + NextConsensus util.Uint160 // Padding that is fixed to 1 _ uint8 // Script used to validate the block - Script transaction.Witness `json:"script"` + Script transaction.Witness // Hash of this block, created when binary encoded (double SHA256). hash util.Uint256 @@ -45,6 +48,20 @@ type Base struct { verificationHash util.Uint256 } +// baseAux is used to marshal/unmarshal to/from JSON, it's almost the same +// as original Base, but with Nonce and NextConsensus fields differing and +// Hash added. +type baseAux struct { + Hash util.Uint256 `json:"hash"` + Version uint32 `json:"version"` + PrevHash util.Uint256 `json:"previousblockhash"` + MerkleRoot util.Uint256 `json:"merkleroot"` + Timestamp uint64 `json:"time"` + Index uint32 `json:"index"` + NextConsensus string `json:"nextconsensus"` + Witnesses []transaction.Witness `json:"witnesses"` +} + // Verify verifies the integrity of the Base. func (b *Base) Verify() bool { // TODO: Need a persisted blockchain for this. @@ -136,3 +153,48 @@ func (b *Base) decodeHashableFields(br *io.BinReader) { b.createHash() } } + +// MarshalJSON implements json.Marshaler interface. +func (b Base) MarshalJSON() ([]byte, error) { + aux := baseAux{ + Hash: b.Hash(), + Version: b.Version, + PrevHash: b.PrevHash, + MerkleRoot: b.MerkleRoot, + Timestamp: b.Timestamp, + Index: b.Index, + NextConsensus: address.Uint160ToString(b.NextConsensus), + Witnesses: []transaction.Witness{b.Script}, + } + return json.Marshal(aux) +} + +// UnmarshalJSON implements json.Unmarshaler interface. +func (b *Base) UnmarshalJSON(data []byte) error { + var aux = new(baseAux) + var nextC util.Uint160 + + err := json.Unmarshal(data, aux) + if err != nil { + return err + } + + nextC, err = address.StringToUint160(aux.NextConsensus) + if err != nil { + return err + } + if len(aux.Witnesses) != 1 { + return errors.New("wrong number of witnesses") + } + b.Version = aux.Version + b.PrevHash = aux.PrevHash + b.MerkleRoot = aux.MerkleRoot + b.Timestamp = aux.Timestamp + b.Index = aux.Index + b.NextConsensus = nextC + b.Script = aux.Witnesses[0] + if !aux.Hash.Equals(b.Hash()) { + return errors.New("json 'hash' doesn't match block hash") + } + return nil +} diff --git a/pkg/core/block/block_test.go b/pkg/core/block/block_test.go index f3461ffbd..b9f6dc72d 100644 --- a/pkg/core/block/block_test.go +++ b/pkg/core/block/block_test.go @@ -172,6 +172,8 @@ func TestBinBlockDecodeEncode(t *testing.T) { data, err := testserdes.EncodeBinary(&b) assert.NoError(t, err) assert.Equal(t, rawtx, hex.EncodeToString(data)) + + testserdes.MarshalUnmarshalJSON(t, &b, new(Block)) } func TestBlockSizeCalculation(t *testing.T) { diff --git a/pkg/core/block/consensus_data.go b/pkg/core/block/consensus_data.go index ef8cc6f95..29211ba26 100644 --- a/pkg/core/block/consensus_data.go +++ b/pkg/core/block/consensus_data.go @@ -1,6 +1,9 @@ package block import ( + "encoding/json" + "strconv" + "github.com/nspcc-dev/neo-go/pkg/crypto/hash" "github.com/nspcc-dev/neo-go/pkg/io" "github.com/nspcc-dev/neo-go/pkg/util" @@ -9,13 +12,19 @@ import ( // ConsensusData represents primary index and nonce of block in the chain. type ConsensusData struct { // Primary index - PrimaryIndex uint32 `json:"primary"` + PrimaryIndex uint32 // Random number - Nonce uint64 `json:"nonce"` + Nonce uint64 // Hash of the ConsensusData (single SHA256) hash util.Uint256 } +// jsonConsensusData is used for JSON I/O of ConsensusData. +type jsonConsensusData struct { + Primary uint32 `json:"primary"` + Nonce string `json:"nonce"` +} + // DecodeBinary implements Serializable interface. func (c *ConsensusData) DecodeBinary(br *io.BinReader) { c.PrimaryIndex = uint32(br.ReadVarUint()) @@ -50,3 +59,28 @@ func (c *ConsensusData) createHash() error { c.hash = hash.Sha256(b) return nil } + +// MarshalJSON implements json.Marshaler interface. +func (c ConsensusData) MarshalJSON() ([]byte, error) { + nonce := strconv.FormatUint(c.Nonce, 16) + for len(nonce) < 16 { + nonce = "0" + nonce + } + return json.Marshal(jsonConsensusData{Primary: c.PrimaryIndex, Nonce: nonce}) +} + +// UnmarshalJSON implements json.Unmarshaler interface. +func (c *ConsensusData) UnmarshalJSON(data []byte) error { + jcd := new(jsonConsensusData) + err := json.Unmarshal(data, jcd) + if err != nil { + return err + } + nonce, err := strconv.ParseUint(jcd.Nonce, 16, 64) + if err != nil { + return err + } + c.PrimaryIndex = jcd.Primary + c.Nonce = nonce + return nil +} diff --git a/pkg/rpc/client/rpc_test.go b/pkg/rpc/client/rpc_test.go index 3bcf13f7a..838a91ad2 100644 --- a/pkg/rpc/client/rpc_test.go +++ b/pkg/rpc/client/rpc_test.go @@ -3,6 +3,7 @@ package client import ( "context" "encoding/hex" + "fmt" "net/http" "net/http/httptest" "strings" @@ -156,12 +157,8 @@ var rpcClientTestCases = map[string][]rpcClientTestCase{ invoke: func(c *Client) (i interface{}, err error) { return c.GetBlockByIndexVerbose(202) }, - serverResponse: `{"id":1,"jsonrpc":"2.0","result":{"hash":"0xcbb73ed9e31dc41a8a222749de475e6ebc2a73b99f73b091a72e0b146110fe86","size":781,"version":0,"nextblockhash":"0x13283c93aec07dc90be3ddd65e2de15e9212f1b3205303f688d6df85129f6b22","previousblockhash":"0x93b540424c1173e487a47582a652686b2885f959ffd895b30e184842403990ef","merkleroot":"0xb8e923148dede20901d6fb225579f6d430cc58f24461d1b0f860ee32abbfcc8d","time":1589300496,"index":202,"consensus_data":{"primary":0,"nonce":"0000000000000457"},"nextconsensus":"AXSvJVzydxXuL9da4GVwK25zdesCrVKkHL","confirmations":6,"script":{"invocation":"0c403620ef8f02d7884c553fb6c54d2fe717cfddd9450886c5fc88a669a29a82fa1a7c715076996567a5a56747f20f10d7e4db071d73b306ccbf17f9a916fcfa1d020c4099e27d87bbb3fb4ce1c77dca85cf3eac46c9c3de87d8022ef7ad2b0a2bb980339293849cf85e5a0a5615ea7bc5bb0a7f28e31f278dc19d628f64c49b888df4c60c40616eefc9286843c2f3f2cf1815988356e409b3f10ffaf60b3468dc0a92dd929cbc8d5da74052c303e7474412f6beaddd551e9056c4e7a5fccdc06107e48f3fe10c40fd2d25d4156e969345c0522669b509e5ced70e4265066eadaf85eea3919d5ded525f8f52d6f0dfa0186c964dd0302fca5bc2dc0540b4ed21085be478c3123996","verification":"130c2102103a7f7dd016558597f7960d27c516a4394fd968b9e65155eb4b013e4040406e0c2102a7bc55fe8684e0119768d104ba30795bdcc86619e864add26156723ed185cd620c2102b3622bf4017bdfe317c58aed5f4c753f206b7db896046fa7d774bbc4bf7f8dc20c2103d90c07df63e690ce77912e10ab51acc944b66860237b608c4f8f8309e71ee699140b413073b3bb"},"tx":[{"txid":"0x96ef00d2efe03101f5b302f7fc3c8fcd22688944bdc83ab99071d77edbb08581","size":254,"type":"ContractTransaction","version":0,"nonce":3,"sender":"ALHF9wsXZVEuCGgmDA6ZNsCLtrb4A1g4yG","sys_fee":"0","net_fee":"0","valid_until_block":1200,"attributes":[],"cosigners":[],"vin":[{"txid":"0x33e045101301854a0e07ff96a92ca1ba9b23c19501f1b7eb15ae9eea07b5f370","vout":0}],"vout":[{"address":"ALHF9wsXZVEuCGgmDA6ZNsCLtrb4A1g4yG","asset":"0x1a5e0e3eac2abced7de9ee2de0820a5c85e63756fcdfc29b82fead86a7c07c78","n":0,"value":"99999000"}],"scripts":[{"invocation":"0c402caebbee911a1f159aa05ab40093d086090a817e837f3f87e8b3e47f6b083649137770f6dda0349ddd611bc47402aca457a89b3b7b0076307ab6a47fd57048eb","verification":"0c2102b3622bf4017bdfe317c58aed5f4c753f206b7db896046fa7d774bbc4bf7f8dc20b410a906ad4"}]}]}}`, + serverResponse: `{"id":1,"jsonrpc":"2.0","result":{"hash":"0xcbb73ed9e31dc41a8a222749de475e6ebc2a73b99f73b091a72e0b146110fe86","size":781,"version":0,"nextblockhash":"0x13283c93aec07dc90be3ddd65e2de15e9212f1b3205303f688d6df85129f6b22","previousblockhash":"0x93b540424c1173e487a47582a652686b2885f959ffd895b30e184842403990ef","merkleroot":"0xb8e923148dede20901d6fb225579f6d430cc58f24461d1b0f860ee32abbfcc8d","time":1589300496,"index":202,"consensus_data":{"primary":0,"nonce":"0000000000000457"},"nextconsensus":"AXSvJVzydxXuL9da4GVwK25zdesCrVKkHL","confirmations":6,"witnesses":[{"invocation":"0c403620ef8f02d7884c553fb6c54d2fe717cfddd9450886c5fc88a669a29a82fa1a7c715076996567a5a56747f20f10d7e4db071d73b306ccbf17f9a916fcfa1d020c4099e27d87bbb3fb4ce1c77dca85cf3eac46c9c3de87d8022ef7ad2b0a2bb980339293849cf85e5a0a5615ea7bc5bb0a7f28e31f278dc19d628f64c49b888df4c60c40616eefc9286843c2f3f2cf1815988356e409b3f10ffaf60b3468dc0a92dd929cbc8d5da74052c303e7474412f6beaddd551e9056c4e7a5fccdc06107e48f3fe10c40fd2d25d4156e969345c0522669b509e5ced70e4265066eadaf85eea3919d5ded525f8f52d6f0dfa0186c964dd0302fca5bc2dc0540b4ed21085be478c3123996","verification":"130c2102103a7f7dd016558597f7960d27c516a4394fd968b9e65155eb4b013e4040406e0c2102a7bc55fe8684e0119768d104ba30795bdcc86619e864add26156723ed185cd620c2102b3622bf4017bdfe317c58aed5f4c753f206b7db896046fa7d774bbc4bf7f8dc20c2103d90c07df63e690ce77912e10ab51acc944b66860237b608c4f8f8309e71ee699140b413073b3bb"}],"tx":[{"txid":"0x96ef00d2efe03101f5b302f7fc3c8fcd22688944bdc83ab99071d77edbb08581","size":254,"type":"ContractTransaction","version":0,"nonce":3,"sender":"ALHF9wsXZVEuCGgmDA6ZNsCLtrb4A1g4yG","sys_fee":"0","net_fee":"0","valid_until_block":1200,"attributes":[],"cosigners":[],"vin":[{"txid":"0x33e045101301854a0e07ff96a92ca1ba9b23c19501f1b7eb15ae9eea07b5f370","vout":0}],"vout":[{"address":"ALHF9wsXZVEuCGgmDA6ZNsCLtrb4A1g4yG","asset":"0x1a5e0e3eac2abced7de9ee2de0820a5c85e63756fcdfc29b82fead86a7c07c78","n":0,"value":"99999000"}],"scripts":[{"invocation":"0c402caebbee911a1f159aa05ab40093d086090a817e837f3f87e8b3e47f6b083649137770f6dda0349ddd611bc47402aca457a89b3b7b0076307ab6a47fd57048eb","verification":"0c2102b3622bf4017bdfe317c58aed5f4c753f206b7db896046fa7d774bbc4bf7f8dc20b410a906ad4"}]}]}}`, result: func(c *Client) interface{} { - hash, err := util.Uint256DecodeStringLE("cbb73ed9e31dc41a8a222749de475e6ebc2a73b99f73b091a72e0b146110fe86") - if err != nil { - panic(err) - } nextBlockHash, err := util.Uint256DecodeStringLE("13283c93aec07dc90be3ddd65e2de15e9212f1b3205303f688d6df85129f6b22") if err != nil { panic(err) @@ -227,28 +224,47 @@ var rpcClientTestCases = map[string][]rpcClientTestCase{ }, } + var nonce uint64 + i, err := fmt.Sscanf("0000000000000457", "%016x", &nonce) + if i != 1 { + panic("can't decode nonce") + } + if err != nil { + panic(err) + } + nextCon, err := address.StringToUint160("AXSvJVzydxXuL9da4GVwK25zdesCrVKkHL") + if err != nil { + panic(err) + } + blck := &block.Block{ + Base: block.Base{ + Version: 0, + PrevHash: prevBlockHash, + MerkleRoot: merkleRoot, + Timestamp: 1589300496, + Index: 202, + NextConsensus: nextCon, + Script: transaction.Witness{ + InvocationScript: invScript, + VerificationScript: verifScript, + }, + }, + ConsensusData: block.ConsensusData{ + PrimaryIndex: 0, + Nonce: nonce, + }, + Transactions: []*transaction.Transaction{tx}, + } // Update hashes for correct result comparison. _ = tx.Hash() + _ = blck.Hash() return &result.Block{ - Hash: hash, - Size: 781, - Version: 0, - NextBlockHash: &nextBlockHash, - PreviousBlockHash: prevBlockHash, - MerkleRoot: merkleRoot, - Time: 1589300496, - Index: 202, - NextConsensus: "AXSvJVzydxXuL9da4GVwK25zdesCrVKkHL", - Confirmations: 6, - ConsensusData: result.ConsensusData{ - PrimaryIndex: 0, - Nonce: "0000000000000457", + Block: blck, + BlockMetadata: result.BlockMetadata{ + Size: 781, + Confirmations: 6, + NextBlockHash: &nextBlockHash, }, - Script: transaction.Witness{ - InvocationScript: invScript, - VerificationScript: verifScript, - }, - Tx: []*transaction.Transaction{tx}, } }, }, @@ -283,12 +299,8 @@ var rpcClientTestCases = map[string][]rpcClientTestCase{ } return c.GetBlockByHashVerbose(hash) }, - serverResponse: `{"id":1,"jsonrpc":"2.0","result":{"hash":"0xcbb73ed9e31dc41a8a222749de475e6ebc2a73b99f73b091a72e0b146110fe86","size":781,"version":0,"nextblockhash":"0x13283c93aec07dc90be3ddd65e2de15e9212f1b3205303f688d6df85129f6b22","previousblockhash":"0x93b540424c1173e487a47582a652686b2885f959ffd895b30e184842403990ef","merkleroot":"0xb8e923148dede20901d6fb225579f6d430cc58f24461d1b0f860ee32abbfcc8d","time":1589300496,"index":202,"consensus_data":{"primary":0,"nonce":"0000000000000457"},"nextconsensus":"AXSvJVzydxXuL9da4GVwK25zdesCrVKkHL","confirmations":6,"script":{"invocation":"0c403620ef8f02d7884c553fb6c54d2fe717cfddd9450886c5fc88a669a29a82fa1a7c715076996567a5a56747f20f10d7e4db071d73b306ccbf17f9a916fcfa1d020c4099e27d87bbb3fb4ce1c77dca85cf3eac46c9c3de87d8022ef7ad2b0a2bb980339293849cf85e5a0a5615ea7bc5bb0a7f28e31f278dc19d628f64c49b888df4c60c40616eefc9286843c2f3f2cf1815988356e409b3f10ffaf60b3468dc0a92dd929cbc8d5da74052c303e7474412f6beaddd551e9056c4e7a5fccdc06107e48f3fe10c40fd2d25d4156e969345c0522669b509e5ced70e4265066eadaf85eea3919d5ded525f8f52d6f0dfa0186c964dd0302fca5bc2dc0540b4ed21085be478c3123996","verification":"130c2102103a7f7dd016558597f7960d27c516a4394fd968b9e65155eb4b013e4040406e0c2102a7bc55fe8684e0119768d104ba30795bdcc86619e864add26156723ed185cd620c2102b3622bf4017bdfe317c58aed5f4c753f206b7db896046fa7d774bbc4bf7f8dc20c2103d90c07df63e690ce77912e10ab51acc944b66860237b608c4f8f8309e71ee699140b413073b3bb"},"tx":[{"txid":"0x96ef00d2efe03101f5b302f7fc3c8fcd22688944bdc83ab99071d77edbb08581","size":254,"type":"ContractTransaction","version":0,"nonce":3,"sender":"ALHF9wsXZVEuCGgmDA6ZNsCLtrb4A1g4yG","sys_fee":"0","net_fee":"0","valid_until_block":1200,"attributes":[],"cosigners":[],"vin":[{"txid":"0x33e045101301854a0e07ff96a92ca1ba9b23c19501f1b7eb15ae9eea07b5f370","vout":0}],"vout":[{"address":"ALHF9wsXZVEuCGgmDA6ZNsCLtrb4A1g4yG","asset":"0x1a5e0e3eac2abced7de9ee2de0820a5c85e63756fcdfc29b82fead86a7c07c78","n":0,"value":"99999000"}],"scripts":[{"invocation":"0c402caebbee911a1f159aa05ab40093d086090a817e837f3f87e8b3e47f6b083649137770f6dda0349ddd611bc47402aca457a89b3b7b0076307ab6a47fd57048eb","verification":"0c2102b3622bf4017bdfe317c58aed5f4c753f206b7db896046fa7d774bbc4bf7f8dc20b410a906ad4"}]}]}}`, + serverResponse: `{"id":1,"jsonrpc":"2.0","result":{"hash":"0xcbb73ed9e31dc41a8a222749de475e6ebc2a73b99f73b091a72e0b146110fe86","size":781,"version":0,"nextblockhash":"0x13283c93aec07dc90be3ddd65e2de15e9212f1b3205303f688d6df85129f6b22","previousblockhash":"0x93b540424c1173e487a47582a652686b2885f959ffd895b30e184842403990ef","merkleroot":"0xb8e923148dede20901d6fb225579f6d430cc58f24461d1b0f860ee32abbfcc8d","time":1589300496,"index":202,"consensus_data":{"primary":0,"nonce":"0000000000000457"},"nextconsensus":"AXSvJVzydxXuL9da4GVwK25zdesCrVKkHL","confirmations":6,"witnesses":[{"invocation":"0c403620ef8f02d7884c553fb6c54d2fe717cfddd9450886c5fc88a669a29a82fa1a7c715076996567a5a56747f20f10d7e4db071d73b306ccbf17f9a916fcfa1d020c4099e27d87bbb3fb4ce1c77dca85cf3eac46c9c3de87d8022ef7ad2b0a2bb980339293849cf85e5a0a5615ea7bc5bb0a7f28e31f278dc19d628f64c49b888df4c60c40616eefc9286843c2f3f2cf1815988356e409b3f10ffaf60b3468dc0a92dd929cbc8d5da74052c303e7474412f6beaddd551e9056c4e7a5fccdc06107e48f3fe10c40fd2d25d4156e969345c0522669b509e5ced70e4265066eadaf85eea3919d5ded525f8f52d6f0dfa0186c964dd0302fca5bc2dc0540b4ed21085be478c3123996","verification":"130c2102103a7f7dd016558597f7960d27c516a4394fd968b9e65155eb4b013e4040406e0c2102a7bc55fe8684e0119768d104ba30795bdcc86619e864add26156723ed185cd620c2102b3622bf4017bdfe317c58aed5f4c753f206b7db896046fa7d774bbc4bf7f8dc20c2103d90c07df63e690ce77912e10ab51acc944b66860237b608c4f8f8309e71ee699140b413073b3bb"}],"tx":[{"txid":"0x96ef00d2efe03101f5b302f7fc3c8fcd22688944bdc83ab99071d77edbb08581","size":254,"type":"ContractTransaction","version":0,"nonce":3,"sender":"ALHF9wsXZVEuCGgmDA6ZNsCLtrb4A1g4yG","sys_fee":"0","net_fee":"0","valid_until_block":1200,"attributes":[],"cosigners":[],"vin":[{"txid":"0x33e045101301854a0e07ff96a92ca1ba9b23c19501f1b7eb15ae9eea07b5f370","vout":0}],"vout":[{"address":"ALHF9wsXZVEuCGgmDA6ZNsCLtrb4A1g4yG","asset":"0x1a5e0e3eac2abced7de9ee2de0820a5c85e63756fcdfc29b82fead86a7c07c78","n":0,"value":"99999000"}],"scripts":[{"invocation":"0c402caebbee911a1f159aa05ab40093d086090a817e837f3f87e8b3e47f6b083649137770f6dda0349ddd611bc47402aca457a89b3b7b0076307ab6a47fd57048eb","verification":"0c2102b3622bf4017bdfe317c58aed5f4c753f206b7db896046fa7d774bbc4bf7f8dc20b410a906ad4"}]}]}}`, result: func(c *Client) interface{} { - hash, err := util.Uint256DecodeStringLE("cbb73ed9e31dc41a8a222749de475e6ebc2a73b99f73b091a72e0b146110fe86") - if err != nil { - panic(err) - } nextBlockHash, err := util.Uint256DecodeStringLE("13283c93aec07dc90be3ddd65e2de15e9212f1b3205303f688d6df85129f6b22") if err != nil { panic(err) @@ -354,28 +366,47 @@ var rpcClientTestCases = map[string][]rpcClientTestCase{ }, } + var nonce uint64 + i, err := fmt.Sscanf("0000000000000457", "%016x", &nonce) + if i != 1 { + panic("can't decode nonce") + } + if err != nil { + panic(err) + } + nextCon, err := address.StringToUint160("AXSvJVzydxXuL9da4GVwK25zdesCrVKkHL") + if err != nil { + panic(err) + } + blck := &block.Block{ + Base: block.Base{ + Version: 0, + PrevHash: prevBlockHash, + MerkleRoot: merkleRoot, + Timestamp: 1589300496, + Index: 202, + NextConsensus: nextCon, + Script: transaction.Witness{ + InvocationScript: invScript, + VerificationScript: verifScript, + }, + }, + ConsensusData: block.ConsensusData{ + PrimaryIndex: 0, + Nonce: nonce, + }, + Transactions: []*transaction.Transaction{tx}, + } // Update hashes for correct result comparison. _ = tx.Hash() + _ = blck.Hash() return &result.Block{ - Hash: hash, - Size: 781, - Version: 0, - NextBlockHash: &nextBlockHash, - PreviousBlockHash: prevBlockHash, - MerkleRoot: merkleRoot, - Time: 1589300496, - Index: 202, - NextConsensus: "AXSvJVzydxXuL9da4GVwK25zdesCrVKkHL", - Confirmations: 6, - ConsensusData: result.ConsensusData{ - PrimaryIndex: 0, - Nonce: "0000000000000457", + Block: blck, + BlockMetadata: result.BlockMetadata{ + Size: 781, + Confirmations: 6, + NextBlockHash: &nextBlockHash, }, - Script: transaction.Witness{ - InvocationScript: invScript, - VerificationScript: verifScript, - }, - Tx: []*transaction.Transaction{tx}, } }, }, diff --git a/pkg/rpc/client/wsclient_test.go b/pkg/rpc/client/wsclient_test.go index 494d31072..55224cf35 100644 --- a/pkg/rpc/client/wsclient_test.go +++ b/pkg/rpc/client/wsclient_test.go @@ -117,7 +117,7 @@ func TestWSClientEvents(t *testing.T) { `{"jsonrpc":"2.0","method":"transaction_executed","params":[{"txid":"0xe1cd5e57e721d2a2e05fb1f08721b12057b25ab1dd7fd0f33ee1639932fdfad7","executions":[{"trigger":"Application","contract":"0x0000000000000000000000000000000000000000","vmstate":"HALT","gas_consumed":"2.291","stack":[],"notifications":[{"contract":"0x1b4357bff5a01bdf2a6581247cf9ed1e24629176","state":{"type":"Array","value":[{"type":"ByteArray","value":"636f6e74726163742063616c6c"},{"type":"ByteArray","value":"7472616e73666572"},{"type":"Array","value":[{"type":"ByteArray","value":"769162241eedf97c2481652adf1ba0f5bf57431b"},{"type":"ByteArray","value":"316e851039019d39dfc2c37d6c3fee19fd580987"},{"type":"Integer","value":"1000"}]}]}},{"contract":"0x1b4357bff5a01bdf2a6581247cf9ed1e24629176","state":{"type":"Array","value":[{"type":"ByteArray","value":"7472616e73666572"},{"type":"ByteArray","value":"769162241eedf97c2481652adf1ba0f5bf57431b"},{"type":"ByteArray","value":"316e851039019d39dfc2c37d6c3fee19fd580987"},{"type":"Integer","value":"1000"}]}}]}]}]}`, `{"jsonrpc":"2.0","method":"notification_from_execution","params":[{"contract":"0x1b4357bff5a01bdf2a6581247cf9ed1e24629176","state":{"type":"Array","value":[{"type":"ByteArray","value":"636f6e74726163742063616c6c"},{"type":"ByteArray","value":"7472616e73666572"},{"type":"Array","value":[{"type":"ByteArray","value":"769162241eedf97c2481652adf1ba0f5bf57431b"},{"type":"ByteArray","value":"316e851039019d39dfc2c37d6c3fee19fd580987"},{"type":"Integer","value":"1000"}]}]}}]}`, `{"jsonrpc":"2.0","method":"transaction_added","params":[{"txid":"0xe1cd5e57e721d2a2e05fb1f08721b12057b25ab1dd7fd0f33ee1639932fdfad7","size":277,"type":"InvocationTransaction","version":1,"nonce":9,"sender":"ALHF9wsXZVEuCGgmDA6ZNsCLtrb4A1g4yG","sys_fee":"0","net_fee":"0.0037721","valid_until_block":1200,"attributes":[],"cosigners":[{"account":"0x870958fd19ee3f6c7dc3c2df399d013910856e31","scopes":1}],"vin":[],"vout":[],"scripts":[{"invocation":"0c4027727296b84853c5d9e07fb8a40e885246ae25641383b16eefbe92027ecb1635b794aacf6bbfc3e828c73829b14791c483d19eb758b57638e3191393dbf2d288","verification":"0c2102b3622bf4017bdfe317c58aed5f4c753f206b7db896046fa7d774bbc4bf7f8dc20b410a906ad4"}],"script":"01e8030c14316e851039019d39dfc2c37d6c3fee19fd5809870c14769162241eedf97c2481652adf1ba0f5bf57431b13c00c087472616e736665720c14769162241eedf97c2481652adf1ba0f5bf57431b41627d5b5238"}]}`, - `{"jsonrpc":"2.0","method":"block_added","params":[{"version":0,"previousblockhash":"0x04f7580b111ec75f0ce68d3a9fd70a0544b4521b4a98541694d8575c548b759e","merkleroot":"0xb2c7230ebee4cb83bc03afadbba413e6bca8fcdeaf9c077bea060918da0e52a1","time":1590006200,"height":207,"next_consensus":"0x4138145d67638db97b402ee0b6751ef16253ecab","script":{"invocation":"0c4063429fca5ff75c964d9e38179c75978e33f8174d91a780c2e825265cf2447281594afdd5f3e216dcaf5ff0693aec83f415996cf224454495495f6bd0a4c5d08f0c4099680903a954278580d8533121c2cd3e53a089817b6a784901ec06178a60b5f1da6e70422bdcadc89029767e08d66ce4180b99334cb2d42f42e4216394af15920c4067d5e362189e48839a24e187c59d46f5d9db862c8a029777f1548b19632bfdc73ad373827ed02369f925e89c2303b64e6b9838dca229949b9b9d3bd4c0c3ed8f0c4021d4c00d4522805883f1db929554441bcbbee127c48f6b7feeeb69a72a78c7f0a75011663e239c0820ef903f36168f42936de10f0ef20681cb735a4b53d0390f","verification":"130c2102103a7f7dd016558597f7960d27c516a4394fd968b9e65155eb4b013e4040406e0c2102a7bc55fe8684e0119768d104ba30795bdcc86619e864add26156723ed185cd620c2102b3622bf4017bdfe317c58aed5f4c753f206b7db896046fa7d774bbc4bf7f8dc20c2103d90c07df63e690ce77912e10ab51acc944b66860237b608c4f8f8309e71ee699140b413073b3bb"},"consensus_data":{"primary":0,"nonce":1111},"tx":[{"txid":"0xf736cd91ab84062a21a09b424346b241987f6245ffe8c2b2db39d595c3c222f7","size":204,"type":"InvocationTransaction","version":1,"nonce":8,"sender":"ALHF9wsXZVEuCGgmDA6ZNsCLtrb4A1g4yG","sys_fee":"0","net_fee":"0.0030421","valid_until_block":1200,"attributes":[],"cosigners":[],"vin":[],"vout":[],"scripts":[{"invocation":"0c4016e7a112742409cdfaad89dcdbcb52c94c5c1a69dfe5d8b999649eaaa787e31ca496d1734d6ea606c749ad36e9a88892240ae59e0efa7f544e0692124898d512","verification":"0c2102b3622bf4017bdfe317c58aed5f4c753f206b7db896046fa7d774bbc4bf7f8dc20b410a906ad4"}],"script":"10c00c04696e69740c14769162241eedf97c2481652adf1ba0f5bf57431b41627d5b52"},{"txid":"0xe1cd5e57e721d2a2e05fb1f08721b12057b25ab1dd7fd0f33ee1639932fdfad7","size":277,"type":"InvocationTransaction","version":1,"nonce":9,"sender":"ALHF9wsXZVEuCGgmDA6ZNsCLtrb4A1g4yG","sys_fee":"0","net_fee":"0.0037721","valid_until_block":1200,"attributes":[],"cosigners":[{"account":"0x870958fd19ee3f6c7dc3c2df399d013910856e31","scopes":1}],"vin":[],"vout":[],"scripts":[{"invocation":"0c4027727296b84853c5d9e07fb8a40e885246ae25641383b16eefbe92027ecb1635b794aacf6bbfc3e828c73829b14791c483d19eb758b57638e3191393dbf2d288","verification":"0c2102b3622bf4017bdfe317c58aed5f4c753f206b7db896046fa7d774bbc4bf7f8dc20b410a906ad4"}],"script":"01e8030c14316e851039019d39dfc2c37d6c3fee19fd5809870c14769162241eedf97c2481652adf1ba0f5bf57431b13c00c087472616e736665720c14769162241eedf97c2481652adf1ba0f5bf57431b41627d5b5238"}]}]}`, + `{"jsonrpc":"2.0","method":"block_added","params":[{"hash":"0x239fea00c54c2f6812612874183b72bef4473fcdf68bf8da08d74fd5b6cab030","version":0,"previousblockhash":"0x04f7580b111ec75f0ce68d3a9fd70a0544b4521b4a98541694d8575c548b759e","merkleroot":"0xb2c7230ebee4cb83bc03afadbba413e6bca8fcdeaf9c077bea060918da0e52a1","time":1590006200,"index":207,"nextconsensus":"AXSvJVzydxXuL9da4GVwK25zdesCrVKkHL","witnesses":[{"invocation":"0c4063429fca5ff75c964d9e38179c75978e33f8174d91a780c2e825265cf2447281594afdd5f3e216dcaf5ff0693aec83f415996cf224454495495f6bd0a4c5d08f0c4099680903a954278580d8533121c2cd3e53a089817b6a784901ec06178a60b5f1da6e70422bdcadc89029767e08d66ce4180b99334cb2d42f42e4216394af15920c4067d5e362189e48839a24e187c59d46f5d9db862c8a029777f1548b19632bfdc73ad373827ed02369f925e89c2303b64e6b9838dca229949b9b9d3bd4c0c3ed8f0c4021d4c00d4522805883f1db929554441bcbbee127c48f6b7feeeb69a72a78c7f0a75011663e239c0820ef903f36168f42936de10f0ef20681cb735a4b53d0390f","verification":"130c2102103a7f7dd016558597f7960d27c516a4394fd968b9e65155eb4b013e4040406e0c2102a7bc55fe8684e0119768d104ba30795bdcc86619e864add26156723ed185cd620c2102b3622bf4017bdfe317c58aed5f4c753f206b7db896046fa7d774bbc4bf7f8dc20c2103d90c07df63e690ce77912e10ab51acc944b66860237b608c4f8f8309e71ee699140b413073b3bb"}],"consensus_data":{"primary":0,"nonce":"0000000000000457"},"tx":[{"txid":"0xf736cd91ab84062a21a09b424346b241987f6245ffe8c2b2db39d595c3c222f7","size":204,"type":"InvocationTransaction","version":1,"nonce":8,"sender":"ALHF9wsXZVEuCGgmDA6ZNsCLtrb4A1g4yG","sys_fee":"0","net_fee":"0.0030421","valid_until_block":1200,"attributes":[],"cosigners":[],"vin":[],"vout":[],"scripts":[{"invocation":"0c4016e7a112742409cdfaad89dcdbcb52c94c5c1a69dfe5d8b999649eaaa787e31ca496d1734d6ea606c749ad36e9a88892240ae59e0efa7f544e0692124898d512","verification":"0c2102b3622bf4017bdfe317c58aed5f4c753f206b7db896046fa7d774bbc4bf7f8dc20b410a906ad4"}],"script":"10c00c04696e69740c14769162241eedf97c2481652adf1ba0f5bf57431b41627d5b52"},{"txid":"0xe1cd5e57e721d2a2e05fb1f08721b12057b25ab1dd7fd0f33ee1639932fdfad7","size":277,"type":"InvocationTransaction","version":1,"nonce":9,"sender":"ALHF9wsXZVEuCGgmDA6ZNsCLtrb4A1g4yG","sys_fee":"0","net_fee":"0.0037721","valid_until_block":1200,"attributes":[],"cosigners":[{"account":"0x870958fd19ee3f6c7dc3c2df399d013910856e31","scopes":1}],"vin":[],"vout":[],"scripts":[{"invocation":"0c4027727296b84853c5d9e07fb8a40e885246ae25641383b16eefbe92027ecb1635b794aacf6bbfc3e828c73829b14791c483d19eb758b57638e3191393dbf2d288","verification":"0c2102b3622bf4017bdfe317c58aed5f4c753f206b7db896046fa7d774bbc4bf7f8dc20b410a906ad4"}],"script":"01e8030c14316e851039019d39dfc2c37d6c3fee19fd5809870c14769162241eedf97c2481652adf1ba0f5bf57431b13c00c087472616e736665720c14769162241eedf97c2481652adf1ba0f5bf57431b41627d5b5238"}]}]}`, `{"jsonrpc":"2.0","method":"event_missed","params":[]}`, } srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { diff --git a/pkg/rpc/response/result/block.go b/pkg/rpc/response/result/block.go index 3e18d6133..d9517a757 100644 --- a/pkg/rpc/response/result/block.go +++ b/pkg/rpc/response/result/block.go @@ -1,65 +1,40 @@ package result import ( - "fmt" + "encoding/json" + "errors" "github.com/nspcc-dev/neo-go/pkg/core/block" "github.com/nspcc-dev/neo-go/pkg/core/blockchainer" - "github.com/nspcc-dev/neo-go/pkg/core/transaction" - "github.com/nspcc-dev/neo-go/pkg/encoding/address" "github.com/nspcc-dev/neo-go/pkg/io" "github.com/nspcc-dev/neo-go/pkg/util" ) type ( - // ConsensusData is a wrapper for block.ConsensusData - ConsensusData struct { - PrimaryIndex uint32 `json:"primary"` - Nonce string `json:"nonce"` - } - // Block wrapper used for the representation of // block.Block / block.Base on the RPC Server. Block struct { - Hash util.Uint256 `json:"hash"` - Size int `json:"size"` - Version uint32 `json:"version"` - NextBlockHash *util.Uint256 `json:"nextblockhash,omitempty"` - PreviousBlockHash util.Uint256 `json:"previousblockhash"` - MerkleRoot util.Uint256 `json:"merkleroot"` - Time uint64 `json:"time"` - Index uint32 `json:"index"` - ConsensusData ConsensusData `json:"consensus_data"` - NextConsensus string `json:"nextconsensus"` + *block.Block + BlockMetadata + } - Confirmations uint32 `json:"confirmations"` - - Script transaction.Witness `json:"script"` - - Tx []*transaction.Transaction `json:"tx"` + // BlockMetadata is an additional metadata added to standard + // block.Block. + BlockMetadata struct { + Size int `json:"size"` + NextBlockHash *util.Uint256 `json:"nextblockhash,omitempty"` + Confirmations uint32 `json:"confirmations"` } ) // NewBlock creates a new Block wrapper. func NewBlock(b *block.Block, chain blockchainer.Blockchainer) Block { res := Block{ - Version: b.Version, - Hash: b.Hash(), - Size: io.GetVarSize(b), - PreviousBlockHash: b.PrevHash, - MerkleRoot: b.MerkleRoot, - Time: b.Timestamp, - Index: b.Index, - ConsensusData: ConsensusData{ - PrimaryIndex: b.ConsensusData.PrimaryIndex, - Nonce: fmt.Sprintf("%016x", b.ConsensusData.Nonce), + Block: b, + BlockMetadata: BlockMetadata{ + Size: io.GetVarSize(b), + Confirmations: chain.BlockHeight() - b.Index - 1, }, - NextConsensus: address.Uint160ToString(b.NextConsensus), - Confirmations: chain.BlockHeight() - b.Index - 1, - - Script: b.Script, - - Tx: b.Transactions, } hash := chain.GetHeaderHash(int(b.Index) + 1) @@ -69,3 +44,44 @@ func NewBlock(b *block.Block, chain blockchainer.Blockchainer) Block { return res } + +// MarshalJSON implements json.Marshaler interface. +func (b Block) MarshalJSON() ([]byte, error) { + output, err := json.Marshal(b.BlockMetadata) + if err != nil { + return nil, err + } + baseBytes, err := json.Marshal(b.Block) + if err != nil { + return nil, err + } + + // We have to keep both "fields" at the same level in json in order to + // match C# API, so there's no way to marshall Block correctly with + // standard json.Marshaller tool. + if output[len(output)-1] != '}' || baseBytes[0] != '{' { + return nil, errors.New("can't merge internal jsons") + } + output[len(output)-1] = ',' + output = append(output, baseBytes[1:]...) + return output, nil +} + +// UnmarshalJSON implements json.Unmarshaler interface. +func (b *Block) UnmarshalJSON(data []byte) error { + // As block.Block and BlockMetadata are at the same level in json, + // do unmarshalling separately for both structs. + meta := new(BlockMetadata) + base := new(block.Block) + err := json.Unmarshal(data, meta) + if err != nil { + return err + } + err = json.Unmarshal(data, base) + if err != nil { + return err + } + b.Block = base + b.BlockMetadata = *meta + return nil +} diff --git a/pkg/rpc/server/server_test.go b/pkg/rpc/server/server_test.go index f36d6c651..5af4908c1 100644 --- a/pkg/rpc/server/server_test.go +++ b/pkg/rpc/server/server_test.go @@ -384,9 +384,8 @@ var rpcTestCases = map[string][]rpcTestCase{ block, err := e.chain.GetBlock(e.chain.GetHeaderHash(3)) require.NoErrorf(t, err, "could not get block") - assert.Equal(t, block.Hash(), res.Hash) - for i := range res.Tx { - tx := res.Tx[i] + assert.Equal(t, block.Hash(), res.Hash()) + for i, tx := range res.Transactions { require.Equal(t, transaction.ContractType, tx.Type) actualTx := block.Transactions[i] diff --git a/pkg/rpc/server/subscription_test.go b/pkg/rpc/server/subscription_test.go index 162674549..ef0ba8155 100644 --- a/pkg/rpc/server/subscription_test.go +++ b/pkg/rpc/server/subscription_test.go @@ -228,7 +228,7 @@ func TestFilteredSubscriptions(t *testing.T) { resp := getNotification(t, respMsgs) rmap := resp.Payload[0].(map[string]interface{}) if resp.Event == response.BlockEventID { - index := rmap["height"].(float64) + index := rmap["index"].(float64) if uint32(index) == lastBlock { break } From 5432530df447fd2f01db31d135bc79a21f9d4a7f Mon Sep 17 00:00:00 2001 From: Roman Khimov Date: Mon, 25 May 2020 16:41:39 +0300 Subject: [PATCH 17/20] result: use witnesses field in result.Header JSON Follow new Neo 3.0 standard for these outputs. --- pkg/rpc/client/rpc_test.go | 6 +++--- pkg/rpc/response/result/block_header.go | 24 ++++++++++++------------ pkg/rpc/server/server_test.go | 2 +- 3 files changed, 16 insertions(+), 16 deletions(-) diff --git a/pkg/rpc/client/rpc_test.go b/pkg/rpc/client/rpc_test.go index 838a91ad2..feaaca5c8 100644 --- a/pkg/rpc/client/rpc_test.go +++ b/pkg/rpc/client/rpc_test.go @@ -469,7 +469,7 @@ var rpcClientTestCases = map[string][]rpcClientTestCase{ } return c.GetBlockHeaderVerbose(hash) }, - serverResponse: `{"id":1,"jsonrpc":"2.0","result":{"hash":"0xe93d17a52967f9e69314385482bf86f85260e811b46bf4d4b261a7f4135a623c","size":442,"version":0,"previousblockhash":"0x996e37358dc369912041f966f8c5d8d3a8255ba5dcbd3447f8a82b55db869099","merkleroot":"0xcb6ddb5f99d6af4c94a6c396d5294472f2eebc91a2c933e0f527422296fa9fb2","time":1541215200,"index":1,"nonce":"51b484a2fe49ed4d","nextconsensus":"AZ81H31DMWzbSnFDLFkzh9vHwaDLayV7fU","script":{"invocation":"40356a91d94e398170e47447d6a0f60aa5470e209782a5452403115a49166db3e1c4a3898122db19f779c30f8ccd0b7d401acdf71eda340655e4ae5237a64961bf4034dd47955e5a71627dafc39dd92999140e9eaeec6b11dbb2b313efa3f1093ed915b4455e199c69ec53778f94ffc236b92f8b97fff97a1f6bbb3770c0c0b3844a40fbe743bd5c90b2f5255e0b073281d7aeb2fb516572f36bec8446bcc37ac755cbf10d08b16c95644db1b2dddc2df5daa377880b20198fc7b967ac6e76474b22df","verification":"532102103a7f7dd016558597f7960d27c516a4394fd968b9e65155eb4b013e4040406e2102a7bc55fe8684e0119768d104ba30795bdcc86619e864add26156723ed185cd622102b3622bf4017bdfe317c58aed5f4c753f206b7db896046fa7d774bbc4bf7f8dc22103d90c07df63e690ce77912e10ab51acc944b66860237b608c4f8f8309e71ee69954ae"},"confirmations":20061,"nextblockhash":"0xcc37d5bc460e72c9423015cb8d579c13e7b03b93bfaa1a23cf4fa777988e035f"}}`, + serverResponse: `{"id":1,"jsonrpc":"2.0","result":{"hash":"0xe93d17a52967f9e69314385482bf86f85260e811b46bf4d4b261a7f4135a623c","size":442,"version":0,"previousblockhash":"0x996e37358dc369912041f966f8c5d8d3a8255ba5dcbd3447f8a82b55db869099","merkleroot":"0xcb6ddb5f99d6af4c94a6c396d5294472f2eebc91a2c933e0f527422296fa9fb2","time":1541215200,"index":1,"nonce":"51b484a2fe49ed4d","nextconsensus":"AZ81H31DMWzbSnFDLFkzh9vHwaDLayV7fU","witnesses":[{"invocation":"40356a91d94e398170e47447d6a0f60aa5470e209782a5452403115a49166db3e1c4a3898122db19f779c30f8ccd0b7d401acdf71eda340655e4ae5237a64961bf4034dd47955e5a71627dafc39dd92999140e9eaeec6b11dbb2b313efa3f1093ed915b4455e199c69ec53778f94ffc236b92f8b97fff97a1f6bbb3770c0c0b3844a40fbe743bd5c90b2f5255e0b073281d7aeb2fb516572f36bec8446bcc37ac755cbf10d08b16c95644db1b2dddc2df5daa377880b20198fc7b967ac6e76474b22df","verification":"532102103a7f7dd016558597f7960d27c516a4394fd968b9e65155eb4b013e4040406e2102a7bc55fe8684e0119768d104ba30795bdcc86619e864add26156723ed185cd622102b3622bf4017bdfe317c58aed5f4c753f206b7db896046fa7d774bbc4bf7f8dc22103d90c07df63e690ce77912e10ab51acc944b66860237b608c4f8f8309e71ee69954ae"}],"confirmations":20061,"nextblockhash":"0xcc37d5bc460e72c9423015cb8d579c13e7b03b93bfaa1a23cf4fa777988e035f"}}`, result: func(c *Client) interface{} { hash, err := util.Uint256DecodeStringLE("e93d17a52967f9e69314385482bf86f85260e811b46bf4d4b261a7f4135a623c") if err != nil { @@ -506,10 +506,10 @@ var rpcClientTestCases = map[string][]rpcClientTestCase{ Index: 1, NextConsensus: "AZ81H31DMWzbSnFDLFkzh9vHwaDLayV7fU", Confirmations: 20061, - Script: transaction.Witness{ + Witnesses: []transaction.Witness{{ InvocationScript: invScript, VerificationScript: verifScript, - }, + }}, } }, }, diff --git a/pkg/rpc/response/result/block_header.go b/pkg/rpc/response/result/block_header.go index 1eeabd4eb..6e087883e 100644 --- a/pkg/rpc/response/result/block_header.go +++ b/pkg/rpc/response/result/block_header.go @@ -13,17 +13,17 @@ type ( // Header wrapper used for the representation of // block header on the RPC Server. Header struct { - Hash util.Uint256 `json:"hash"` - Size int `json:"size"` - Version uint32 `json:"version"` - PrevBlockHash util.Uint256 `json:"previousblockhash"` - MerkleRoot util.Uint256 `json:"merkleroot"` - Timestamp uint64 `json:"time"` - Index uint32 `json:"index"` - NextConsensus string `json:"nextconsensus"` - Script transaction.Witness `json:"script"` - Confirmations uint32 `json:"confirmations"` - NextBlockHash *util.Uint256 `json:"nextblockhash,omitempty"` + Hash util.Uint256 `json:"hash"` + Size int `json:"size"` + Version uint32 `json:"version"` + PrevBlockHash util.Uint256 `json:"previousblockhash"` + MerkleRoot util.Uint256 `json:"merkleroot"` + Timestamp uint64 `json:"time"` + Index uint32 `json:"index"` + NextConsensus string `json:"nextconsensus"` + Witnesses []transaction.Witness `json:"witnesses"` + Confirmations uint32 `json:"confirmations"` + NextBlockHash *util.Uint256 `json:"nextblockhash,omitempty"` } ) @@ -38,7 +38,7 @@ func NewHeader(h *block.Header, chain blockchainer.Blockchainer) Header { Timestamp: h.Timestamp, Index: h.Index, NextConsensus: address.Uint160ToString(h.NextConsensus), - Script: h.Script, + Witnesses: []transaction.Witness{h.Script}, Confirmations: chain.BlockHeight() - h.Index + 1, } diff --git a/pkg/rpc/server/server_test.go b/pkg/rpc/server/server_test.go index 5af4908c1..1bcb382b5 100644 --- a/pkg/rpc/server/server_test.go +++ b/pkg/rpc/server/server_test.go @@ -1041,7 +1041,7 @@ func testRPCProtocol(t *testing.T, doRPCCall func(string, string, *testing.T) [] Timestamp: hdr.Timestamp, Index: hdr.Index, NextConsensus: address.Uint160ToString(hdr.NextConsensus), - Script: hdr.Script, + Witnesses: []transaction.Witness{hdr.Script}, Confirmations: e.chain.BlockHeight() - hdr.Index + 1, NextBlockHash: &nextHash, } From 29e66925aacf48b5f300de3260fac8dd936bc506 Mon Sep 17 00:00:00 2001 From: Roman Khimov Date: Thu, 14 May 2020 10:29:11 +0300 Subject: [PATCH 18/20] rpc/client: deduplicate block/header tests a bit The same data is copied at least twice here. Also use this block for header test making it a bit more convenient. --- pkg/rpc/client/rpc_test.go | 381 +++++++++++++------------------------ 1 file changed, 127 insertions(+), 254 deletions(-) diff --git a/pkg/rpc/client/rpc_test.go b/pkg/rpc/client/rpc_test.go index feaaca5c8..c3b92a43a 100644 --- a/pkg/rpc/client/rpc_test.go +++ b/pkg/rpc/client/rpc_test.go @@ -31,6 +31,117 @@ type rpcClientTestCase struct { check func(t *testing.T, c *Client, result interface{}) } +// getResultBlock202 returns data for block number 1 which is used by several tests. +func getResultBlock202() *result.Block { + nextBlockHash, err := util.Uint256DecodeStringLE("13283c93aec07dc90be3ddd65e2de15e9212f1b3205303f688d6df85129f6b22") + if err != nil { + panic(err) + } + prevBlockHash, err := util.Uint256DecodeStringLE("93b540424c1173e487a47582a652686b2885f959ffd895b30e184842403990ef") + if err != nil { + panic(err) + } + merkleRoot, err := util.Uint256DecodeStringLE("b8e923148dede20901d6fb225579f6d430cc58f24461d1b0f860ee32abbfcc8d") + if err != nil { + panic(err) + } + invScript, err := hex.DecodeString("0c403620ef8f02d7884c553fb6c54d2fe717cfddd9450886c5fc88a669a29a82fa1a7c715076996567a5a56747f20f10d7e4db071d73b306ccbf17f9a916fcfa1d020c4099e27d87bbb3fb4ce1c77dca85cf3eac46c9c3de87d8022ef7ad2b0a2bb980339293849cf85e5a0a5615ea7bc5bb0a7f28e31f278dc19d628f64c49b888df4c60c40616eefc9286843c2f3f2cf1815988356e409b3f10ffaf60b3468dc0a92dd929cbc8d5da74052c303e7474412f6beaddd551e9056c4e7a5fccdc06107e48f3fe10c40fd2d25d4156e969345c0522669b509e5ced70e4265066eadaf85eea3919d5ded525f8f52d6f0dfa0186c964dd0302fca5bc2dc0540b4ed21085be478c3123996") + if err != nil { + panic(err) + } + verifScript, err := hex.DecodeString("130c2102103a7f7dd016558597f7960d27c516a4394fd968b9e65155eb4b013e4040406e0c2102a7bc55fe8684e0119768d104ba30795bdcc86619e864add26156723ed185cd620c2102b3622bf4017bdfe317c58aed5f4c753f206b7db896046fa7d774bbc4bf7f8dc20c2103d90c07df63e690ce77912e10ab51acc944b66860237b608c4f8f8309e71ee699140b413073b3bb") + if err != nil { + panic(err) + } + sender, err := address.StringToUint160("ALHF9wsXZVEuCGgmDA6ZNsCLtrb4A1g4yG") + if err != nil { + panic(err) + } + txInvScript, err := hex.DecodeString("0c402caebbee911a1f159aa05ab40093d086090a817e837f3f87e8b3e47f6b083649137770f6dda0349ddd611bc47402aca457a89b3b7b0076307ab6a47fd57048eb") + if err != nil { + panic(err) + } + txVerifScript, err := hex.DecodeString("0c2102b3622bf4017bdfe317c58aed5f4c753f206b7db896046fa7d774bbc4bf7f8dc20b410a906ad4") + if err != nil { + panic(err) + } + vin, err := util.Uint256DecodeStringLE("33e045101301854a0e07ff96a92ca1ba9b23c19501f1b7eb15ae9eea07b5f370") + if err != nil { + panic(err) + } + outAddress, err := address.StringToUint160("ALHF9wsXZVEuCGgmDA6ZNsCLtrb4A1g4yG") + if err != nil { + panic(err) + } + tx := transaction.NewContractTX() + tx.Nonce = 3 + tx.ValidUntilBlock = 1200 + tx.Sender = sender + tx.Scripts = []transaction.Witness{ + { + InvocationScript: txInvScript, + VerificationScript: txVerifScript, + }, + } + tx.Inputs = []transaction.Input{ + { + PrevHash: vin, + PrevIndex: 0, + }, + } + tx.Outputs = []transaction.Output{ + { + AssetID: core.GoverningTokenID(), + Amount: util.Fixed8FromInt64(99999000), + ScriptHash: outAddress, + Position: 0, + }, + } + + var nonce uint64 + i, err := fmt.Sscanf("0000000000000457", "%016x", &nonce) + if i != 1 { + panic("can't decode nonce") + } + if err != nil { + panic(err) + } + nextCon, err := address.StringToUint160("AXSvJVzydxXuL9da4GVwK25zdesCrVKkHL") + if err != nil { + panic(err) + } + blck := &block.Block{ + Base: block.Base{ + Version: 0, + PrevHash: prevBlockHash, + MerkleRoot: merkleRoot, + Timestamp: 1589300496, + Index: 202, + NextConsensus: nextCon, + Script: transaction.Witness{ + InvocationScript: invScript, + VerificationScript: verifScript, + }, + }, + ConsensusData: block.ConsensusData{ + PrimaryIndex: 0, + Nonce: nonce, + }, + Transactions: []*transaction.Transaction{tx}, + } + // Update hashes for correct result comparison. + _ = tx.Hash() + _ = blck.Hash() + return &result.Block{ + Block: blck, + BlockMetadata: result.BlockMetadata{ + Size: 781, + Confirmations: 6, + NextBlockHash: &nextBlockHash, + }, + } +} + // rpcClientTestCases contains `serverResponse` json data fetched from examples // published in official C# JSON-RPC API v2.10.3 reference // (see https://docs.neo.org/docs/en-us/reference/rpc/latest-version/api.html) @@ -159,113 +270,7 @@ var rpcClientTestCases = map[string][]rpcClientTestCase{ }, serverResponse: `{"id":1,"jsonrpc":"2.0","result":{"hash":"0xcbb73ed9e31dc41a8a222749de475e6ebc2a73b99f73b091a72e0b146110fe86","size":781,"version":0,"nextblockhash":"0x13283c93aec07dc90be3ddd65e2de15e9212f1b3205303f688d6df85129f6b22","previousblockhash":"0x93b540424c1173e487a47582a652686b2885f959ffd895b30e184842403990ef","merkleroot":"0xb8e923148dede20901d6fb225579f6d430cc58f24461d1b0f860ee32abbfcc8d","time":1589300496,"index":202,"consensus_data":{"primary":0,"nonce":"0000000000000457"},"nextconsensus":"AXSvJVzydxXuL9da4GVwK25zdesCrVKkHL","confirmations":6,"witnesses":[{"invocation":"0c403620ef8f02d7884c553fb6c54d2fe717cfddd9450886c5fc88a669a29a82fa1a7c715076996567a5a56747f20f10d7e4db071d73b306ccbf17f9a916fcfa1d020c4099e27d87bbb3fb4ce1c77dca85cf3eac46c9c3de87d8022ef7ad2b0a2bb980339293849cf85e5a0a5615ea7bc5bb0a7f28e31f278dc19d628f64c49b888df4c60c40616eefc9286843c2f3f2cf1815988356e409b3f10ffaf60b3468dc0a92dd929cbc8d5da74052c303e7474412f6beaddd551e9056c4e7a5fccdc06107e48f3fe10c40fd2d25d4156e969345c0522669b509e5ced70e4265066eadaf85eea3919d5ded525f8f52d6f0dfa0186c964dd0302fca5bc2dc0540b4ed21085be478c3123996","verification":"130c2102103a7f7dd016558597f7960d27c516a4394fd968b9e65155eb4b013e4040406e0c2102a7bc55fe8684e0119768d104ba30795bdcc86619e864add26156723ed185cd620c2102b3622bf4017bdfe317c58aed5f4c753f206b7db896046fa7d774bbc4bf7f8dc20c2103d90c07df63e690ce77912e10ab51acc944b66860237b608c4f8f8309e71ee699140b413073b3bb"}],"tx":[{"txid":"0x96ef00d2efe03101f5b302f7fc3c8fcd22688944bdc83ab99071d77edbb08581","size":254,"type":"ContractTransaction","version":0,"nonce":3,"sender":"ALHF9wsXZVEuCGgmDA6ZNsCLtrb4A1g4yG","sys_fee":"0","net_fee":"0","valid_until_block":1200,"attributes":[],"cosigners":[],"vin":[{"txid":"0x33e045101301854a0e07ff96a92ca1ba9b23c19501f1b7eb15ae9eea07b5f370","vout":0}],"vout":[{"address":"ALHF9wsXZVEuCGgmDA6ZNsCLtrb4A1g4yG","asset":"0x1a5e0e3eac2abced7de9ee2de0820a5c85e63756fcdfc29b82fead86a7c07c78","n":0,"value":"99999000"}],"scripts":[{"invocation":"0c402caebbee911a1f159aa05ab40093d086090a817e837f3f87e8b3e47f6b083649137770f6dda0349ddd611bc47402aca457a89b3b7b0076307ab6a47fd57048eb","verification":"0c2102b3622bf4017bdfe317c58aed5f4c753f206b7db896046fa7d774bbc4bf7f8dc20b410a906ad4"}]}]}}`, result: func(c *Client) interface{} { - nextBlockHash, err := util.Uint256DecodeStringLE("13283c93aec07dc90be3ddd65e2de15e9212f1b3205303f688d6df85129f6b22") - if err != nil { - panic(err) - } - prevBlockHash, err := util.Uint256DecodeStringLE("93b540424c1173e487a47582a652686b2885f959ffd895b30e184842403990ef") - if err != nil { - panic(err) - } - merkleRoot, err := util.Uint256DecodeStringLE("b8e923148dede20901d6fb225579f6d430cc58f24461d1b0f860ee32abbfcc8d") - if err != nil { - panic(err) - } - invScript, err := hex.DecodeString("0c403620ef8f02d7884c553fb6c54d2fe717cfddd9450886c5fc88a669a29a82fa1a7c715076996567a5a56747f20f10d7e4db071d73b306ccbf17f9a916fcfa1d020c4099e27d87bbb3fb4ce1c77dca85cf3eac46c9c3de87d8022ef7ad2b0a2bb980339293849cf85e5a0a5615ea7bc5bb0a7f28e31f278dc19d628f64c49b888df4c60c40616eefc9286843c2f3f2cf1815988356e409b3f10ffaf60b3468dc0a92dd929cbc8d5da74052c303e7474412f6beaddd551e9056c4e7a5fccdc06107e48f3fe10c40fd2d25d4156e969345c0522669b509e5ced70e4265066eadaf85eea3919d5ded525f8f52d6f0dfa0186c964dd0302fca5bc2dc0540b4ed21085be478c3123996") - if err != nil { - panic(err) - } - verifScript, err := hex.DecodeString("130c2102103a7f7dd016558597f7960d27c516a4394fd968b9e65155eb4b013e4040406e0c2102a7bc55fe8684e0119768d104ba30795bdcc86619e864add26156723ed185cd620c2102b3622bf4017bdfe317c58aed5f4c753f206b7db896046fa7d774bbc4bf7f8dc20c2103d90c07df63e690ce77912e10ab51acc944b66860237b608c4f8f8309e71ee699140b413073b3bb") - if err != nil { - panic(err) - } - sender, err := address.StringToUint160("ALHF9wsXZVEuCGgmDA6ZNsCLtrb4A1g4yG") - if err != nil { - panic(err) - } - txInvScript, err := hex.DecodeString("0c402caebbee911a1f159aa05ab40093d086090a817e837f3f87e8b3e47f6b083649137770f6dda0349ddd611bc47402aca457a89b3b7b0076307ab6a47fd57048eb") - if err != nil { - panic(err) - } - txVerifScript, err := hex.DecodeString("0c2102b3622bf4017bdfe317c58aed5f4c753f206b7db896046fa7d774bbc4bf7f8dc20b410a906ad4") - if err != nil { - panic(err) - } - vin, err := util.Uint256DecodeStringLE("33e045101301854a0e07ff96a92ca1ba9b23c19501f1b7eb15ae9eea07b5f370") - if err != nil { - panic(err) - } - outAddress, err := address.StringToUint160("ALHF9wsXZVEuCGgmDA6ZNsCLtrb4A1g4yG") - if err != nil { - panic(err) - } - tx := transaction.NewContractTX() - tx.Nonce = 3 - tx.ValidUntilBlock = 1200 - tx.Sender = sender - tx.Scripts = []transaction.Witness{ - { - InvocationScript: txInvScript, - VerificationScript: txVerifScript, - }, - } - tx.Inputs = []transaction.Input{ - { - PrevHash: vin, - PrevIndex: 0, - }, - } - tx.Outputs = []transaction.Output{ - { - AssetID: core.GoverningTokenID(), - Amount: util.Fixed8FromInt64(99999000), - ScriptHash: outAddress, - Position: 0, - }, - } - - var nonce uint64 - i, err := fmt.Sscanf("0000000000000457", "%016x", &nonce) - if i != 1 { - panic("can't decode nonce") - } - if err != nil { - panic(err) - } - nextCon, err := address.StringToUint160("AXSvJVzydxXuL9da4GVwK25zdesCrVKkHL") - if err != nil { - panic(err) - } - blck := &block.Block{ - Base: block.Base{ - Version: 0, - PrevHash: prevBlockHash, - MerkleRoot: merkleRoot, - Timestamp: 1589300496, - Index: 202, - NextConsensus: nextCon, - Script: transaction.Witness{ - InvocationScript: invScript, - VerificationScript: verifScript, - }, - }, - ConsensusData: block.ConsensusData{ - PrimaryIndex: 0, - Nonce: nonce, - }, - Transactions: []*transaction.Transaction{tx}, - } - // Update hashes for correct result comparison. - _ = tx.Hash() - _ = blck.Hash() - return &result.Block{ - Block: blck, - BlockMetadata: result.BlockMetadata{ - Size: 781, - Confirmations: 6, - NextBlockHash: &nextBlockHash, - }, - } + return getResultBlock202() }, }, { @@ -301,113 +306,7 @@ var rpcClientTestCases = map[string][]rpcClientTestCase{ }, serverResponse: `{"id":1,"jsonrpc":"2.0","result":{"hash":"0xcbb73ed9e31dc41a8a222749de475e6ebc2a73b99f73b091a72e0b146110fe86","size":781,"version":0,"nextblockhash":"0x13283c93aec07dc90be3ddd65e2de15e9212f1b3205303f688d6df85129f6b22","previousblockhash":"0x93b540424c1173e487a47582a652686b2885f959ffd895b30e184842403990ef","merkleroot":"0xb8e923148dede20901d6fb225579f6d430cc58f24461d1b0f860ee32abbfcc8d","time":1589300496,"index":202,"consensus_data":{"primary":0,"nonce":"0000000000000457"},"nextconsensus":"AXSvJVzydxXuL9da4GVwK25zdesCrVKkHL","confirmations":6,"witnesses":[{"invocation":"0c403620ef8f02d7884c553fb6c54d2fe717cfddd9450886c5fc88a669a29a82fa1a7c715076996567a5a56747f20f10d7e4db071d73b306ccbf17f9a916fcfa1d020c4099e27d87bbb3fb4ce1c77dca85cf3eac46c9c3de87d8022ef7ad2b0a2bb980339293849cf85e5a0a5615ea7bc5bb0a7f28e31f278dc19d628f64c49b888df4c60c40616eefc9286843c2f3f2cf1815988356e409b3f10ffaf60b3468dc0a92dd929cbc8d5da74052c303e7474412f6beaddd551e9056c4e7a5fccdc06107e48f3fe10c40fd2d25d4156e969345c0522669b509e5ced70e4265066eadaf85eea3919d5ded525f8f52d6f0dfa0186c964dd0302fca5bc2dc0540b4ed21085be478c3123996","verification":"130c2102103a7f7dd016558597f7960d27c516a4394fd968b9e65155eb4b013e4040406e0c2102a7bc55fe8684e0119768d104ba30795bdcc86619e864add26156723ed185cd620c2102b3622bf4017bdfe317c58aed5f4c753f206b7db896046fa7d774bbc4bf7f8dc20c2103d90c07df63e690ce77912e10ab51acc944b66860237b608c4f8f8309e71ee699140b413073b3bb"}],"tx":[{"txid":"0x96ef00d2efe03101f5b302f7fc3c8fcd22688944bdc83ab99071d77edbb08581","size":254,"type":"ContractTransaction","version":0,"nonce":3,"sender":"ALHF9wsXZVEuCGgmDA6ZNsCLtrb4A1g4yG","sys_fee":"0","net_fee":"0","valid_until_block":1200,"attributes":[],"cosigners":[],"vin":[{"txid":"0x33e045101301854a0e07ff96a92ca1ba9b23c19501f1b7eb15ae9eea07b5f370","vout":0}],"vout":[{"address":"ALHF9wsXZVEuCGgmDA6ZNsCLtrb4A1g4yG","asset":"0x1a5e0e3eac2abced7de9ee2de0820a5c85e63756fcdfc29b82fead86a7c07c78","n":0,"value":"99999000"}],"scripts":[{"invocation":"0c402caebbee911a1f159aa05ab40093d086090a817e837f3f87e8b3e47f6b083649137770f6dda0349ddd611bc47402aca457a89b3b7b0076307ab6a47fd57048eb","verification":"0c2102b3622bf4017bdfe317c58aed5f4c753f206b7db896046fa7d774bbc4bf7f8dc20b410a906ad4"}]}]}}`, result: func(c *Client) interface{} { - nextBlockHash, err := util.Uint256DecodeStringLE("13283c93aec07dc90be3ddd65e2de15e9212f1b3205303f688d6df85129f6b22") - if err != nil { - panic(err) - } - prevBlockHash, err := util.Uint256DecodeStringLE("93b540424c1173e487a47582a652686b2885f959ffd895b30e184842403990ef") - if err != nil { - panic(err) - } - merkleRoot, err := util.Uint256DecodeStringLE("b8e923148dede20901d6fb225579f6d430cc58f24461d1b0f860ee32abbfcc8d") - if err != nil { - panic(err) - } - invScript, err := hex.DecodeString("0c403620ef8f02d7884c553fb6c54d2fe717cfddd9450886c5fc88a669a29a82fa1a7c715076996567a5a56747f20f10d7e4db071d73b306ccbf17f9a916fcfa1d020c4099e27d87bbb3fb4ce1c77dca85cf3eac46c9c3de87d8022ef7ad2b0a2bb980339293849cf85e5a0a5615ea7bc5bb0a7f28e31f278dc19d628f64c49b888df4c60c40616eefc9286843c2f3f2cf1815988356e409b3f10ffaf60b3468dc0a92dd929cbc8d5da74052c303e7474412f6beaddd551e9056c4e7a5fccdc06107e48f3fe10c40fd2d25d4156e969345c0522669b509e5ced70e4265066eadaf85eea3919d5ded525f8f52d6f0dfa0186c964dd0302fca5bc2dc0540b4ed21085be478c3123996") - if err != nil { - panic(err) - } - verifScript, err := hex.DecodeString("130c2102103a7f7dd016558597f7960d27c516a4394fd968b9e65155eb4b013e4040406e0c2102a7bc55fe8684e0119768d104ba30795bdcc86619e864add26156723ed185cd620c2102b3622bf4017bdfe317c58aed5f4c753f206b7db896046fa7d774bbc4bf7f8dc20c2103d90c07df63e690ce77912e10ab51acc944b66860237b608c4f8f8309e71ee699140b413073b3bb") - if err != nil { - panic(err) - } - sender, err := address.StringToUint160("ALHF9wsXZVEuCGgmDA6ZNsCLtrb4A1g4yG") - if err != nil { - panic(err) - } - txInvScript, err := hex.DecodeString("0c402caebbee911a1f159aa05ab40093d086090a817e837f3f87e8b3e47f6b083649137770f6dda0349ddd611bc47402aca457a89b3b7b0076307ab6a47fd57048eb") - if err != nil { - panic(err) - } - txVerifScript, err := hex.DecodeString("0c2102b3622bf4017bdfe317c58aed5f4c753f206b7db896046fa7d774bbc4bf7f8dc20b410a906ad4") - if err != nil { - panic(err) - } - vin, err := util.Uint256DecodeStringLE("33e045101301854a0e07ff96a92ca1ba9b23c19501f1b7eb15ae9eea07b5f370") - if err != nil { - panic(err) - } - outAddress, err := address.StringToUint160("ALHF9wsXZVEuCGgmDA6ZNsCLtrb4A1g4yG") - if err != nil { - panic(err) - } - tx := transaction.NewContractTX() - tx.Nonce = 3 - tx.ValidUntilBlock = 1200 - tx.Sender = sender - tx.Scripts = []transaction.Witness{ - { - InvocationScript: txInvScript, - VerificationScript: txVerifScript, - }, - } - tx.Inputs = []transaction.Input{ - { - PrevHash: vin, - PrevIndex: 0, - }, - } - tx.Outputs = []transaction.Output{ - { - AssetID: core.GoverningTokenID(), - Amount: util.Fixed8FromInt64(99999000), - ScriptHash: outAddress, - Position: 0, - }, - } - - var nonce uint64 - i, err := fmt.Sscanf("0000000000000457", "%016x", &nonce) - if i != 1 { - panic("can't decode nonce") - } - if err != nil { - panic(err) - } - nextCon, err := address.StringToUint160("AXSvJVzydxXuL9da4GVwK25zdesCrVKkHL") - if err != nil { - panic(err) - } - blck := &block.Block{ - Base: block.Base{ - Version: 0, - PrevHash: prevBlockHash, - MerkleRoot: merkleRoot, - Timestamp: 1589300496, - Index: 202, - NextConsensus: nextCon, - Script: transaction.Witness{ - InvocationScript: invScript, - VerificationScript: verifScript, - }, - }, - ConsensusData: block.ConsensusData{ - PrimaryIndex: 0, - Nonce: nonce, - }, - Transactions: []*transaction.Transaction{tx}, - } - // Update hashes for correct result comparison. - _ = tx.Hash() - _ = blck.Hash() - return &result.Block{ - Block: blck, - BlockMetadata: result.BlockMetadata{ - Size: 781, - Confirmations: 6, - NextBlockHash: &nextBlockHash, - }, - } + return getResultBlock202() }, }, }, @@ -463,53 +362,27 @@ var rpcClientTestCases = map[string][]rpcClientTestCase{ { name: "verbose_positive", invoke: func(c *Client) (i interface{}, err error) { - hash, err := util.Uint256DecodeStringLE("e93d17a52967f9e69314385482bf86f85260e811b46bf4d4b261a7f4135a623c") + hash, err := util.Uint256DecodeStringLE("cbb73ed9e31dc41a8a222749de475e6ebc2a73b99f73b091a72e0b146110fe86") if err != nil { panic(err) } return c.GetBlockHeaderVerbose(hash) }, - serverResponse: `{"id":1,"jsonrpc":"2.0","result":{"hash":"0xe93d17a52967f9e69314385482bf86f85260e811b46bf4d4b261a7f4135a623c","size":442,"version":0,"previousblockhash":"0x996e37358dc369912041f966f8c5d8d3a8255ba5dcbd3447f8a82b55db869099","merkleroot":"0xcb6ddb5f99d6af4c94a6c396d5294472f2eebc91a2c933e0f527422296fa9fb2","time":1541215200,"index":1,"nonce":"51b484a2fe49ed4d","nextconsensus":"AZ81H31DMWzbSnFDLFkzh9vHwaDLayV7fU","witnesses":[{"invocation":"40356a91d94e398170e47447d6a0f60aa5470e209782a5452403115a49166db3e1c4a3898122db19f779c30f8ccd0b7d401acdf71eda340655e4ae5237a64961bf4034dd47955e5a71627dafc39dd92999140e9eaeec6b11dbb2b313efa3f1093ed915b4455e199c69ec53778f94ffc236b92f8b97fff97a1f6bbb3770c0c0b3844a40fbe743bd5c90b2f5255e0b073281d7aeb2fb516572f36bec8446bcc37ac755cbf10d08b16c95644db1b2dddc2df5daa377880b20198fc7b967ac6e76474b22df","verification":"532102103a7f7dd016558597f7960d27c516a4394fd968b9e65155eb4b013e4040406e2102a7bc55fe8684e0119768d104ba30795bdcc86619e864add26156723ed185cd622102b3622bf4017bdfe317c58aed5f4c753f206b7db896046fa7d774bbc4bf7f8dc22103d90c07df63e690ce77912e10ab51acc944b66860237b608c4f8f8309e71ee69954ae"}],"confirmations":20061,"nextblockhash":"0xcc37d5bc460e72c9423015cb8d579c13e7b03b93bfaa1a23cf4fa777988e035f"}}`, + serverResponse: `{"id":1,"jsonrpc":"2.0","result":{"hash":"0xcbb73ed9e31dc41a8a222749de475e6ebc2a73b99f73b091a72e0b146110fe86","size":781,"version":0,"nextblockhash":"0x13283c93aec07dc90be3ddd65e2de15e9212f1b3205303f688d6df85129f6b22","previousblockhash":"0x93b540424c1173e487a47582a652686b2885f959ffd895b30e184842403990ef","merkleroot":"0xb8e923148dede20901d6fb225579f6d430cc58f24461d1b0f860ee32abbfcc8d","time":1589300496,"index":202,"nextconsensus":"AXSvJVzydxXuL9da4GVwK25zdesCrVKkHL","confirmations":6,"witnesses":[{"invocation":"0c403620ef8f02d7884c553fb6c54d2fe717cfddd9450886c5fc88a669a29a82fa1a7c715076996567a5a56747f20f10d7e4db071d73b306ccbf17f9a916fcfa1d020c4099e27d87bbb3fb4ce1c77dca85cf3eac46c9c3de87d8022ef7ad2b0a2bb980339293849cf85e5a0a5615ea7bc5bb0a7f28e31f278dc19d628f64c49b888df4c60c40616eefc9286843c2f3f2cf1815988356e409b3f10ffaf60b3468dc0a92dd929cbc8d5da74052c303e7474412f6beaddd551e9056c4e7a5fccdc06107e48f3fe10c40fd2d25d4156e969345c0522669b509e5ced70e4265066eadaf85eea3919d5ded525f8f52d6f0dfa0186c964dd0302fca5bc2dc0540b4ed21085be478c3123996","verification":"130c2102103a7f7dd016558597f7960d27c516a4394fd968b9e65155eb4b013e4040406e0c2102a7bc55fe8684e0119768d104ba30795bdcc86619e864add26156723ed185cd620c2102b3622bf4017bdfe317c58aed5f4c753f206b7db896046fa7d774bbc4bf7f8dc20c2103d90c07df63e690ce77912e10ab51acc944b66860237b608c4f8f8309e71ee699140b413073b3bb"}]}}`, result: func(c *Client) interface{} { - hash, err := util.Uint256DecodeStringLE("e93d17a52967f9e69314385482bf86f85260e811b46bf4d4b261a7f4135a623c") - if err != nil { - panic(err) - } - nextBlockHash, err := util.Uint256DecodeStringLE("cc37d5bc460e72c9423015cb8d579c13e7b03b93bfaa1a23cf4fa777988e035f") - if err != nil { - panic(err) - } - prevBlockHash, err := util.Uint256DecodeStringLE("996e37358dc369912041f966f8c5d8d3a8255ba5dcbd3447f8a82b55db869099") - if err != nil { - panic(err) - } - merkleRoot, err := util.Uint256DecodeStringLE("cb6ddb5f99d6af4c94a6c396d5294472f2eebc91a2c933e0f527422296fa9fb2") - if err != nil { - panic(err) - } - invScript, err := hex.DecodeString("40356a91d94e398170e47447d6a0f60aa5470e209782a5452403115a49166db3e1c4a3898122db19f779c30f8ccd0b7d401acdf71eda340655e4ae5237a64961bf4034dd47955e5a71627dafc39dd92999140e9eaeec6b11dbb2b313efa3f1093ed915b4455e199c69ec53778f94ffc236b92f8b97fff97a1f6bbb3770c0c0b3844a40fbe743bd5c90b2f5255e0b073281d7aeb2fb516572f36bec8446bcc37ac755cbf10d08b16c95644db1b2dddc2df5daa377880b20198fc7b967ac6e76474b22df") - if err != nil { - panic(err) - } - verifScript, err := hex.DecodeString("532102103a7f7dd016558597f7960d27c516a4394fd968b9e65155eb4b013e4040406e2102a7bc55fe8684e0119768d104ba30795bdcc86619e864add26156723ed185cd622102b3622bf4017bdfe317c58aed5f4c753f206b7db896046fa7d774bbc4bf7f8dc22103d90c07df63e690ce77912e10ab51acc944b66860237b608c4f8f8309e71ee69954ae") - if err != nil { - panic(err) - } + b := getResultBlock202() return &result.Header{ - Hash: hash, - Size: 442, - Version: 0, - NextBlockHash: &nextBlockHash, - PrevBlockHash: prevBlockHash, - MerkleRoot: merkleRoot, - Timestamp: 1541215200, - Index: 1, - NextConsensus: "AZ81H31DMWzbSnFDLFkzh9vHwaDLayV7fU", - Confirmations: 20061, - Witnesses: []transaction.Witness{{ - InvocationScript: invScript, - VerificationScript: verifScript, - }}, + Hash: b.Hash(), + Size: 781, + Version: b.Version, + NextBlockHash: b.NextBlockHash, + PrevBlockHash: b.PrevHash, + MerkleRoot: b.MerkleRoot, + Timestamp: b.Timestamp, + Index: b.Index, + NextConsensus: address.Uint160ToString(b.NextConsensus), + Witnesses: []transaction.Witness{b.Script}, + Confirmations: 6, } }, }, From aca6f2f3ad81ba77ee0c06323f52f454ab6a8d98 Mon Sep 17 00:00:00 2001 From: alexvanin Date: Mon, 18 May 2020 16:23:51 +0300 Subject: [PATCH 19/20] rpc/client: handle client creation error in new wsclient Client returns error if it can't parse endpoint string. WSClient should check client error or there could be panic at `cl.cli = nil` expression. --- pkg/rpc/client/wsclient.go | 4 ++++ pkg/rpc/client/wsclient_test.go | 15 +++++++++++++++ 2 files changed, 19 insertions(+) diff --git a/pkg/rpc/client/wsclient.go b/pkg/rpc/client/wsclient.go index 24363f1dd..3f469a86a 100644 --- a/pkg/rpc/client/wsclient.go +++ b/pkg/rpc/client/wsclient.go @@ -71,6 +71,10 @@ const ( // connection). You need to use websocket URL for it like `ws://1.2.3.4/ws`. func NewWS(ctx context.Context, endpoint string, opts Options) (*WSClient, error) { cl, err := New(ctx, endpoint, opts) + if err != nil { + return nil, err + } + cl.cli = nil dialer := websocket.Dialer{HandshakeTimeout: opts.DialTimeout} diff --git a/pkg/rpc/client/wsclient_test.go b/pkg/rpc/client/wsclient_test.go index 55224cf35..a31a6beb0 100644 --- a/pkg/rpc/client/wsclient_test.go +++ b/pkg/rpc/client/wsclient_test.go @@ -4,6 +4,7 @@ import ( "context" "net/http" "net/http/httptest" + "strings" "testing" "time" @@ -297,3 +298,17 @@ func TestWSFilteredSubscriptions(t *testing.T) { }) } } + +func TestNewWS(t *testing.T) { + srv := initTestServer(t, "") + defer srv.Close() + + t.Run("good", func(t *testing.T) { + _, err := NewWS(context.TODO(), httpURLtoWS(srv.URL), Options{}) + require.NoError(t, err) + }) + t.Run("bad URL", func(t *testing.T) { + _, err := NewWS(context.TODO(), strings.Trim(srv.URL, "http://"), Options{}) + require.Error(t, err) + }) +} From b29db96cdbcd1b6a5f9aacb326f2c44a6a11604e Mon Sep 17 00:00:00 2001 From: Roman Khimov Date: Tue, 19 May 2020 01:01:35 +0300 Subject: [PATCH 20/20] docs: update RPC document, add notifications spec --- docs/notifications.md | 425 ++++++++++++++++++++++++++++++++++++++++++ docs/rpc.md | 35 +++- 2 files changed, 452 insertions(+), 8 deletions(-) create mode 100644 docs/notifications.md diff --git a/docs/notifications.md b/docs/notifications.md new file mode 100644 index 000000000..5d1878ca7 --- /dev/null +++ b/docs/notifications.md @@ -0,0 +1,425 @@ +# Notification subsystem + +Original motivation, requirements and general solution strategy are described +in the issue #895. + +This extension allows a websocket client to subscribe to various events and +receive them as JSON-RPC notifications from the server. + +## Events +Currently supported events: + * new block added + Contents: block. + Filters: primary ID. + * new transaction in the block + Contents: transaction. + Filters: sender and cosigner. + * notification generated during execution + Contents: container hash, contract script hash, stack item. + Filters: contract script hash. + * transaction executed + Contents: application execution result. + Filters: VM state. + +Filters use conjunctional logic. + +## Ordering and persistence guarantees + * new block is only announced after its processing is complete and the chain + is updated to the new height + * no disk-level persistence guarantees are given + * new in-block transaction is announced after block processing, but before + announcing the block itself + * transaction notifications are only announced for successful transactions + * all announcements are being done in the same order they happen on the chain + At first transaction execution is announced, then followed by notifications + generated during this execution, then followed by transaction announcement. + Transaction announcements are ordered the same way they're in the block. + * unsubscription may not cancel pending, but not yet sent events + +## Subscription management + +To receive events clients need to subscribe to them first via `subscribe` +method. Upon successful subscription clients receive subscription ID for +subsequent management of this subscription. Subscription is only valid for +connection lifetime, no long-term client identification is being made. + +Errors are not described down below, but they can be returned as standard +JSON-RPC errors (most often caused by invalid parameters). + +### `subscribe` method + +Parameters: event stream name, stream-specific filter rules hash (can be +omitted if empty). + +Recognized stream names: + * `block_added` + Filter: `primary` as an integer with primary (speaker) node index from + ConsensusData. + * `transaction_added` + Filter: `sender` field containing string with hex-encoded Uint160 (LE + representation) for transaction's `Sender` and/or `cosigner` in the same + format for one of transaction's `Cosigners`. + * `notification_from_execution` + Filter: `contract` field containing string with hex-encoded Uint160 (LE + representation). + * `transaction_executed` + Filter: `state` field containing `HALT` or `FAULT` string for successful + and failed executions respectively. + +Response: returns subscription ID (string) as a result. This ID can be used to +cancel this subscription and has no meaning other than that. + +Example request (subscribe to notifications from contract +0x6293a440ed80a427038e175a507d3def1e04fb67 generated when executing +transactions): + +``` +{ + "jsonrpc": "2.0", + "method": "subscribe", + "params": ["notification_from_execution", {"contract": "6293a440ed80a427038e175a507d3def1e04fb67"}], + "id": 1 +} + +``` + +Example response: + +``` +{ + "jsonrpc": "2.0", + "id": 1, + "result": "55aaff00" +} +``` + +### `unsubscribe` method + +Parameters: subscription ID as a string. + +Response: boolean true. + +Example request (unsubscribe from "55aaff00"): + +``` +{ + "jsonrpc": "2.0", + "method": "unsubscribe", + "params": ["55aaff00"], + "id": 1 +} +``` + +Example response: + +``` +{ + "jsonrpc": "2.0", + "id": 1, + "result": true +} +``` + +## Events + +Events are sent as JSON-RPC notifications from the server with `method` field +being used for notification names. Notification names are identical to stream +names described for `subscribe` method with one important addition for +`event_missed` which can be sent for any subscription to signify that some +events were not delivered (usually when client isn't able to keep up with +event flow). + +Verbose responses for various structures like blocks and transactions are used +to simplify working with notifications on client side. Returned structures +mostly follow the one used by standard Neo RPC calls, but may have some minor +differences. + +If a server-side event matches several subscriptions from one client, it's +only sent once. + +### `block_added` notification +As a first parameter (`params` section) contains block converted to JSON +structure which is similar to verbose `getblock` response but with the +following differences: + * it doesn't have `size` field (you can calculate it client-side) + * it doesn't have `nextblockhash` field (it's supposed to be the latest one + anyway) + * it doesn't have `confirmations` field (see previous) + +No other parameters are sent. + +Example: +``` +{ + "params" : [ + { + "index" : 207, + "time" : 1590006200, + "nextconsensus" : "AXSvJVzydxXuL9da4GVwK25zdesCrVKkHL", + "consensus_data" : { + "primary" : 0, + "nonce" : "0000000000000457" + }, + "previousblockhash" : "0x04f7580b111ec75f0ce68d3a9fd70a0544b4521b4a98541694d8575c548b759e", + "witnesses" : [ + { + "invocation" : "0c4063429fca5ff75c964d9e38179c75978e33f8174d91a780c2e825265cf2447281594afdd5f3e216dcaf5ff0693aec83f415996cf224454495495f6bd0a4c5d08f0c4099680903a954278580d8533121c2cd3e53a089817b6a784901ec06178a60b5f1da6e70422bdcadc89029767e08d66ce4180b99334cb2d42f42e4216394af15920c4067d5e362189e48839a24e187c59d46f5d9db862c8a029777f1548b19632bfdc73ad373827ed02369f925e89c2303b64e6b9838dca229949b9b9d3bd4c0c3ed8f0c4021d4c00d4522805883f1db929554441bcbbee127c48f6b7feeeb69a72a78c7f0a75011663e239c0820ef903f36168f42936de10f0ef20681cb735a4b53d0390f", + "verification" : "130c2102103a7f7dd016558597f7960d27c516a4394fd968b9e65155eb4b013e4040406e0c2102a7bc55fe8684e0119768d104ba30795bdcc86619e864add26156723ed185cd620c2102b3622bf4017bdfe317c58aed5f4c753f206b7db896046fa7d774bbc4bf7f8dc20c2103d90c07df63e690ce77912e10ab51acc944b66860237b608c4f8f8309e71ee699140b413073b3bb" + } + ], + "version" : 0, + "hash" : "0x239fea00c54c2f6812612874183b72bef4473fcdf68bf8da08d74fd5b6cab030", + "tx" : [ + { + "txid" : "0xf736cd91ab84062a21a09b424346b241987f6245ffe8c2b2db39d595c3c222f7", + "scripts" : [ + { + "verification" : "0c2102b3622bf4017bdfe317c58aed5f4c753f206b7db896046fa7d774bbc4bf7f8dc20b410a906ad4", + "invocation" : "0c4016e7a112742409cdfaad89dcdbcb52c94c5c1a69dfe5d8b999649eaaa787e31ca496d1734d6ea606c749ad36e9a88892240ae59e0efa7f544e0692124898d512" + } + ], + "vout" : [], + "cosigners" : [], + "valid_until_block" : 1200, + "nonce" : 8, + "net_fee" : "0.0030421", + "sender" : "ALHF9wsXZVEuCGgmDA6ZNsCLtrb4A1g4yG", + "sys_fee" : "0", + "type" : "InvocationTransaction", + "attributes" : [], + "version" : 1, + "vin" : [], + "size" : 204, + "script" : "10c00c04696e69740c14769162241eedf97c2481652adf1ba0f5bf57431b41627d5b52" + }, + { + "script" : "01e8030c14316e851039019d39dfc2c37d6c3fee19fd5809870c14769162241eedf97c2481652adf1ba0f5bf57431b13c00c087472616e736665720c14769162241eedf97c2481652adf1ba0f5bf57431b41627d5b5238", + "size" : 277, + "attributes" : [], + "version" : 1, + "vin" : [], + "net_fee" : "0.0037721", + "sender" : "ALHF9wsXZVEuCGgmDA6ZNsCLtrb4A1g4yG", + "sys_fee" : "0", + "type" : "InvocationTransaction", + "nonce" : 9, + "cosigners" : [ + { + "scopes" : 1, + "account" : "0x870958fd19ee3f6c7dc3c2df399d013910856e31" + } + ], + "valid_until_block" : 1200, + "scripts" : [ + { + "invocation" : "0c4027727296b84853c5d9e07fb8a40e885246ae25641383b16eefbe92027ecb1635b794aacf6bbfc3e828c73829b14791c483d19eb758b57638e3191393dbf2d288", + "verification" : "0c2102b3622bf4017bdfe317c58aed5f4c753f206b7db896046fa7d774bbc4bf7f8dc20b410a906ad4" + } + ], + "vout" : [], + "txid" : "0xe1cd5e57e721d2a2e05fb1f08721b12057b25ab1dd7fd0f33ee1639932fdfad7" + } + ], + "merkleroot" : "0xb2c7230ebee4cb83bc03afadbba413e6bca8fcdeaf9c077bea060918da0e52a1" + } + ], + "jsonrpc" : "2.0", + "method" : "block_added" +} +``` + +### `transaction_added` notification + +In the first parameter (`params` section) contains transaction converted to +JSON which is similar to verbose `getrawtransaction` response, but with the +following differences: + * block's metadata is missing (`blockhash`, `confirmations`, `blocktime`) + +No other parameters are sent. + +Example: +``` +{ + "method" : "transaction_added", + "params" : [ + { + "valid_until_block" : 1200, + "version" : 1, + "txid" : "0xe1cd5e57e721d2a2e05fb1f08721b12057b25ab1dd7fd0f33ee1639932fdfad7", + "scripts" : [ + { + "invocation" : "0c4027727296b84853c5d9e07fb8a40e885246ae25641383b16eefbe92027ecb1635b794aacf6bbfc3e828c73829b14791c483d19eb758b57638e3191393dbf2d288", + "verification" : "0c2102b3622bf4017bdfe317c58aed5f4c753f206b7db896046fa7d774bbc4bf7f8dc20b410a906ad4" + } + ], + "sys_fee" : "0", + "sender" : "ALHF9wsXZVEuCGgmDA6ZNsCLtrb4A1g4yG", + "vout" : [], + "net_fee" : "0.0037721", + "size" : 277, + "attributes" : [], + "script" : "01e8030c14316e851039019d39dfc2c37d6c3fee19fd5809870c14769162241eedf97c2481652adf1ba0f5bf57431b13c00c087472616e736665720c14769162241eedf97c2481652adf1ba0f5bf57431b41627d5b5238", + "nonce" : 9, + "vin" : [], + "type" : "InvocationTransaction", + "cosigners" : [ + { + "account" : "0x870958fd19ee3f6c7dc3c2df399d013910856e31", + "scopes" : 1 + } + ] + } + ], + "jsonrpc" : "2.0" +} +``` + +### `notification_from_execution` notification + +Contains three parameters: container hash (hex-encoded LE Uint256 in a +string), contract script hash (hex-encoded LE Uint160 in a string) and stack +item (encoded the same way as `state` field contents for notifications from +`getapplicationlog` response). + +Example: + +``` +{ + "jsonrpc" : "2.0", + "method" : "notification_from_execution", + "params" : [ + { + "state" : { + "value" : [ + { + "value" : "636f6e74726163742063616c6c", + "type" : "ByteArray" + }, + { + "value" : "7472616e73666572", + "type" : "ByteArray" + }, + { + "value" : [ + { + "value" : "769162241eedf97c2481652adf1ba0f5bf57431b", + "type" : "ByteArray" + }, + { + "value" : "316e851039019d39dfc2c37d6c3fee19fd580987", + "type" : "ByteArray" + }, + { + "value" : "1000", + "type" : "Integer" + } + ], + "type" : "Array" + } + ], + "type" : "Array" + }, + "contract" : "0x1b4357bff5a01bdf2a6581247cf9ed1e24629176" + } + ] +} +``` + +### `transaction_executed` notification + +Contains the same result as from `getapplicationlog` method in the first +parameter and no other parameters. One difference from `getapplicationlog` is +that it always contains zero in the `contract` field. + +Example: +``` +{ + "method" : "transaction_executed", + "params" : [ + { + "txid" : "0xe1cd5e57e721d2a2e05fb1f08721b12057b25ab1dd7fd0f33ee1639932fdfad7", + "executions" : [ + { + "trigger" : "Application", + "gas_consumed" : "2.291", + "contract" : "0x0000000000000000000000000000000000000000", + "stack" : [], + "notifications" : [ + { + "state" : { + "type" : "Array", + "value" : [ + { + "value" : "636f6e74726163742063616c6c", + "type" : "ByteArray" + }, + { + "type" : "ByteArray", + "value" : "7472616e73666572" + }, + { + "value" : [ + { + "value" : "769162241eedf97c2481652adf1ba0f5bf57431b", + "type" : "ByteArray" + }, + { + "type" : "ByteArray", + "value" : "316e851039019d39dfc2c37d6c3fee19fd580987" + }, + { + "value" : "1000", + "type" : "Integer" + } + ], + "type" : "Array" + } + ] + }, + "contract" : "0x1b4357bff5a01bdf2a6581247cf9ed1e24629176" + }, + { + "contract" : "0x1b4357bff5a01bdf2a6581247cf9ed1e24629176", + "state" : { + "value" : [ + { + "value" : "7472616e73666572", + "type" : "ByteArray" + }, + { + "value" : "769162241eedf97c2481652adf1ba0f5bf57431b", + "type" : "ByteArray" + }, + { + "value" : "316e851039019d39dfc2c37d6c3fee19fd580987", + "type" : "ByteArray" + }, + { + "value" : "1000", + "type" : "Integer" + } + ], + "type" : "Array" + } + } + ], + "vmstate" : "HALT" + } + ] + } + ], + "jsonrpc" : "2.0" +} +``` + +### `event_missed` notification + +Never has any parameters. Example: + +``` +{ + "jsonrpc": "2.0", + "method": "event_missed", + "params": [] +} +``` diff --git a/docs/rpc.md b/docs/rpc.md index 9cae46a16..ba025c07d 100644 --- a/docs/rpc.md +++ b/docs/rpc.md @@ -65,6 +65,18 @@ which would yield the response: | `submitblock` | | `validateaddress` | +#### Implementation notices + +##### `invokefunction` and `invoke` + +neo-go's implementation of `invokefunction` and `invoke` does not return `tx` +field in the answer because that requires signing the transaction with some +key in the server which doesn't fit the model of our node-client interactions. +Lacking this signature the transaction is almost useless, so there is no point +in returning it. + +Both methods also don't currently support arrays in function parameters. + ### Unsupported methods Methods listed down below are not going to be supported for various reasons @@ -86,17 +98,24 @@ and we're not accepting issues related to them. | `sendmany` | Not applicable to neo-go, see `claimgas` comment | | `sendtoaddress` | Not applicable to neo-go, see `claimgas` comment | -#### Implementation notices +### Extensions -##### `invokefunction` and `invoke` +Some additional extensions are implemented as a part of this RPC server. -neo-go's implementation of `invokefunction` and `invoke` does not return `tx` -field in the answer because that requires signing the transaction with some -key in the server which doesn't fit the model of our node-client interactions. -Lacking this signature the transaction is almost useless, so there is no point -in returning it. +#### Websocket server -Both methods also don't currently support arrays in function parameters. +This server accepts websocket connections on `ws://$BASE_URL/ws` address. You +can use it to perform regular RPC calls over websockets (it's supposed to be a +little faster than going regular HTTP route) and you can also use it for +additional functionality provided only via websockets (like notifications). + +#### Notification subsystem + +Notification subsystem consists of two additional RPC methods (`subscribe` and +`unsubscribe` working only over websocket connection) that allow to subscribe +to various blockchain events (with simple event filtering) and receive them on +the client as JSON-RPC notifications. More details on that are written in the +[notifications specification](notifications.md). ## Reference