core: implement basic policying support, fix #370

Implement mempool and consensus block creation policies, almost the same as
SimplePolicy plugin for C# node provides with two caveats:
 * HighPriorityTxType is not configured and hardcoded to ClaimType
 * BlockedAccounts are not supported

Other than that it allows us to run successfuly as testnet CN, previously our
proposals were rejected because we were proposing blocks with oversized
transactions (that are rejected by PoolTx() now).

Mainnet and testnet configuration files are updated accordingly, but privnet
is left as is with no limits.

Configuration is currently attached to the Blockchain and so is the code that
does policying, it may be moved somewhere in the future, but it works for
now.

config/config.go

@@ -46,7 +46,7 @@ type (
 		AddressVersion          byte      `yaml:"AddressVersion"`
 		SecondsPerBlock         int       `yaml:"SecondsPerBlock"`
 		LowPriorityThreshold    float64   `yaml:"LowPriorityThreshold"`
-		MaxTransactionsPerBlock int64     `yaml:"MaxTransactionsPerBlock"`
+		MaxTransactionsPerBlock int       `yaml:"MaxTransactionsPerBlock"`
 		MemPoolSize             int       `yaml:"MemPoolSize"`
 		StandbyValidators       []string  `yaml:"StandbyValidators"`
 		SeedList                []string  `yaml:"SeedList"`
@@ -59,6 +59,13 @@ type (
 		FreeGasLimit util.Fixed8 `yaml:"FreeGasLimit"`
 		// SaveStorageBatch enables storage batch saving before every persist.
 		SaveStorageBatch bool `yaml:"SaveStorageBatch"`
+		// Maximum number of low priority transactions accepted into block.
+		MaxFreeTransactionsPerBlock int `yaml:"MaxFreeTransactionsPerBlock"`
+		// Maximum size of low priority transaction in bytes.
+		MaxFreeTransactionSize int `yaml:"MaxFreeTransactionSize"`
+		// FeePerExtraByte sets the expected per-byte fee for
+		// transactions exceeding the MaxFreeTransactionSize.
+		FeePerExtraByte float64 `yaml:"FeePerExtraByte"`
 	}
 
 	// SystemFee fees related to system.

config/protocol.mainnet.yml

@@ -31,6 +31,10 @@ ProtocolConfiguration:
   VerifyBlocks: true
   VerifyTransactions: false
   FreeGasLimit: 10.0
+  MaxTransactionsPerBlock: 500
+  MaxFreeTransactionsPerBlock: 20
+  MaxFreeTransactionSize: 1024
+  FeePerExtraByte: 0.00001
 
 ApplicationConfiguration:
   # LogPath could be set up in case you need stdout logs to some proper file.

config/protocol.testnet.yml

@@ -31,6 +31,10 @@ ProtocolConfiguration:
   VerifyBlocks: true
   VerifyTransactions: false
   FreeGasLimit: 10.0
+  MaxTransactionsPerBlock: 500
+  MaxFreeTransactionsPerBlock: 20
+  MaxFreeTransactionSize: 1024
+  FeePerExtraByte: 0.00001
 
 ApplicationConfiguration:
   # LogPath could be set up in case you need stdout logs to some proper file.

pkg/consensus/consensus.go

@@ -430,6 +430,10 @@ func (s *service) getVerifiedTx(count int) []block.Transaction {
 		txx = pool.GetVerifiedTransactions()
 	}
 
+	if len(txx) > 0 {
+		txx = s.Config.Chain.ApplyPolicyToTxSet(txx)
+	}
+
 	res := make([]block.Transaction, len(txx)+1)
 	var netFee util.Fixed8
 	for i := range txx {

pkg/core/blockchain.go

@@ -49,6 +49,9 @@ var (
 	// ErrOOM is returned when adding transaction to the memory pool because
 	// it reached its full capacity.
 	ErrOOM = errors.New("no space left in the memory pool")
+	// ErrPolicy is returned on attempt to add transaction that doesn't
+	// comply with node's configured policy into the mempool.
+	ErrPolicy = errors.New("not allowed by policy")
 )
 var (
 	genAmount         = []int{8, 7, 6, 5, 4, 3, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1}
@@ -125,6 +128,22 @@ func NewBlockchain(s storage.Store, cfg config.ProtocolConfiguration, log *zap.L
 		cfg.MemPoolSize = defaultMemPoolSize
 		log.Info("mempool size is not set or wrong, setting default value", zap.Int("MemPoolSize", cfg.MemPoolSize))
 	}
+	if cfg.MaxTransactionsPerBlock <= 0 {
+		cfg.MaxTransactionsPerBlock = 0
+		log.Info("MaxTransactionsPerBlock is not set or wrong, setting default value (unlimited)", zap.Int("MaxTransactionsPerBlock", cfg.MaxTransactionsPerBlock))
+	}
+	if cfg.MaxFreeTransactionsPerBlock <= 0 {
+		cfg.MaxFreeTransactionsPerBlock = 0
+		log.Info("MaxFreeTransactionsPerBlock is not set or wrong, setting default value (unlimited)", zap.Int("MaxFreeTransactionsPerBlock", cfg.MaxFreeTransactionsPerBlock))
+	}
+	if cfg.MaxFreeTransactionSize <= 0 {
+		cfg.MaxFreeTransactionSize = 0
+		log.Info("MaxFreeTransactionSize is not set or wrong, setting default value (unlimited)", zap.Int("MaxFreeTransactionSize", cfg.MaxFreeTransactionSize))
+	}
+	if cfg.FeePerExtraByte <= 0 {
+		cfg.FeePerExtraByte = 0
+		log.Info("FeePerExtraByte is not set or wrong, setting default value", zap.Float64("FeePerExtraByte", cfg.FeePerExtraByte))
+	}
 	bc := &Blockchain{
 		config:        cfg,
 		dao:           newDao(s),
@@ -1030,6 +1049,24 @@ func (bc *Blockchain) GetMemPool() *mempool.Pool {
 	return &bc.memPool
 }
 
+// ApplyPolicyToTxSet applies configured policies to given transaction set. It
+// expects slice to be ordered by fee and returns a subslice of it.
+func (bc *Blockchain) ApplyPolicyToTxSet(txes []mempool.TxWithFee) []mempool.TxWithFee {
+	if bc.config.MaxTransactionsPerBlock != 0 && len(txes) > bc.config.MaxTransactionsPerBlock {
+		txes = txes[:bc.config.MaxTransactionsPerBlock]
+	}
+	maxFree := bc.config.MaxFreeTransactionsPerBlock
+	if maxFree != 0 {
+		lowStart := sort.Search(len(txes), func(i int) bool {
+			return bc.IsLowPriority(txes[i].Fee)
+		})
+		if lowStart+maxFree < len(txes) {
+			txes = txes[:lowStart+maxFree]
+		}
+	}
+	return txes
+}
+
 // VerifyBlock verifies block against its current state.
 func (bc *Blockchain) VerifyBlock(block *block.Block) error {
 	prevHeader, err := bc.GetHeader(block.PrevHash)
@@ -1127,6 +1164,18 @@ func (bc *Blockchain) PoolTx(t *transaction.Transaction) error {
 	if err := bc.verifyTx(t, nil); err != nil {
 		return err
 	}
+	// Policying.
+	if t.Type != transaction.ClaimType {
+		txSize := io.GetVarSize(t)
+		maxFree := bc.config.MaxFreeTransactionSize
+		if maxFree != 0 && txSize > maxFree {
+			netFee := bc.NetworkFee(t)
+			if bc.IsLowPriority(netFee) ||
+				netFee < util.Fixed8FromFloat(bc.config.FeePerExtraByte)*util.Fixed8(txSize-maxFree) {
+				return ErrPolicy
+			}
+		}
+	}
 	if err := bc.memPool.Add(t, bc); err != nil {
 		switch err {
 		case mempool.ErrOOM:

pkg/core/blockchainer.go

@@ -15,6 +15,7 @@ import (
 // Blockchainer is an interface that abstract the implementation
 // of the blockchain.
 type Blockchainer interface {
+	ApplyPolicyToTxSet([]mempool.TxWithFee) []mempool.TxWithFee
 	GetConfig() config.ProtocolConfiguration
 	AddHeaders(...*block.Header) error
 	AddBlock(*block.Block) error

pkg/network/helper_test.go

@@ -26,6 +26,9 @@ type testChain struct {
 	blockheight uint32
 }
 
+func (chain testChain) ApplyPolicyToTxSet([]mempool.TxWithFee) []mempool.TxWithFee {
+	panic("TODO")
+}
 func (chain testChain) GetConfig() config.ProtocolConfiguration {
 	panic("TODO")
 }

pkg/network/server.go

@@ -786,15 +786,14 @@ func (s *Server) verifyAndPoolTX(t *transaction.Transaction) RelayReason {
 	if t.Type == transaction.MinerType {
 		return RelayInvalid
 	}
-	// TODO: Implement Plugin.CheckPolicy?
-	//if (!Plugin.CheckPolicy(transaction))
-	// return RelayResultReason.PolicyFail;
 	if err := s.chain.PoolTx(t); err != nil {
 		switch err {
 		case core.ErrAlreadyExists:
 			return RelayAlreadyExists
 		case core.ErrOOM:
 			return RelayOutOfMemory
+		case core.ErrPolicy:
+			return RelayPolicyFail
 		default:
 			return RelayInvalid
 		}