network: introduce peer-to-peer message queue

This one is designed to give more priority to direct nodes communication, that
is that their messaging would have more priority than generic broadcasts. It
should improve consensus process under TX pressure and allow to handle
pings in time (preventing disconnects).

pkg/network/tcp_peer.go

@@ -22,6 +22,7 @@ const (
 	verAckReceived
 
 	requestQueueSize   = 32
+	p2pMsgQueueSize    = 16
 	hpRequestQueueSize = 4
 )
 
@@ -47,9 +48,10 @@ type TCPPeer struct {
 	finale    sync.Once
 	handShake handShakeStage
 
-	done    chan struct{}
-	sendQ   chan []byte
-	hpSendQ chan []byte
+	done     chan struct{}
+	sendQ    chan []byte
+	p2pSendQ chan []byte
+	hpSendQ  chan []byte
 
 	wg sync.WaitGroup
 
@@ -61,41 +63,60 @@ type TCPPeer struct {
 // NewTCPPeer returns a TCPPeer structure based on the given connection.
 func NewTCPPeer(conn net.Conn, s *Server) *TCPPeer {
 	return &TCPPeer{
-		conn:    conn,
-		server:  s,
-		done:    make(chan struct{}),
-		sendQ:   make(chan []byte, requestQueueSize),
-		hpSendQ: make(chan []byte, hpRequestQueueSize),
+		conn:     conn,
+		server:   s,
+		done:     make(chan struct{}),
+		sendQ:    make(chan []byte, requestQueueSize),
+		p2pSendQ: make(chan []byte, p2pMsgQueueSize),
+		hpSendQ:  make(chan []byte, hpRequestQueueSize),
 	}
 }
 
+// putPacketIntoQueue puts given message into the given queue if the peer has
+// done handshaking.
+func (p *TCPPeer) putPacketIntoQueue(queue chan<- []byte, msg []byte) error {
+	if !p.Handshaked() {
+		return errStateMismatch
+	}
+	queue <- msg
+	return nil
+}
+
 // EnqueuePacket implements the Peer interface.
 func (p *TCPPeer) EnqueuePacket(msg []byte) error {
-	if !p.Handshaked() {
-		return errStateMismatch
+	return p.putPacketIntoQueue(p.sendQ, msg)
+}
+
+// putMessageIntoQueue serializes given Message and puts it into given queue if
+// the peer has done handshaking.
+func (p *TCPPeer) putMsgIntoQueue(queue chan<- []byte, msg *Message) error {
+	b, err := msg.Bytes()
+	if err != nil {
+		return err
 	}
-	p.sendQ <- msg
-	return nil
+	return p.putPacketIntoQueue(queue, b)
 }
 
 // EnqueueMessage is a temporary wrapper that sends a message via
 // EnqueuePacket if there is no error in serializing it.
 func (p *TCPPeer) EnqueueMessage(msg *Message) error {
-	b, err := msg.Bytes()
-	if err != nil {
-		return err
-	}
-	return p.EnqueuePacket(b)
+	return p.putMsgIntoQueue(p.sendQ, msg)
+}
+
+// EnqueueP2PPacket implements the Peer interface.
+func (p *TCPPeer) EnqueueP2PPacket(msg []byte) error {
+	return p.putPacketIntoQueue(p.p2pSendQ, msg)
+}
+
+// EnqueueP2PMessage implements the Peer interface.
+func (p *TCPPeer) EnqueueP2PMessage(msg *Message) error {
+	return p.putMsgIntoQueue(p.p2pSendQ, msg)
 }
 
 // EnqueueHPPacket implements the Peer interface. It the peer is not yet
 // handshaked it's a noop.
 func (p *TCPPeer) EnqueueHPPacket(msg []byte) error {
-	if !p.Handshaked() {
-		return errStateMismatch
-	}
-	p.hpSendQ <- msg
-	return nil
+	return p.putPacketIntoQueue(p.hpSendQ, msg)
 }
 
 func (p *TCPPeer) writeMsg(msg *Message) error {
@@ -158,13 +179,24 @@ func (p *TCPPeer) handleQueues() {
 		default:
 		}
 
-		// If there is no message in the hp queue, block until one
+		if msg == nil {
+			// Then look at the p2p queue.
+			select {
+			case <-p.done:
+				return
+			case msg = <-p.hpSendQ:
+			case msg = <-p.p2pSendQ:
+			default:
+			}
+		}
+		// If there is no message in HP or P2P queues, block until one
 		// appears in any of the queues.
 		if msg == nil {
 			select {
 			case <-p.done:
 				return
 			case msg = <-p.hpSendQ:
+			case msg = <-p.p2pSendQ:
 			case msg = <-p.sendQ:
 			}
 		}