From 83306a5c9624c2e6dbad8b6e8c0d9c3cab803ab5 Mon Sep 17 00:00:00 2001 From: Anthony De Meulemeester Date: Thu, 1 Feb 2018 09:00:42 +0100 Subject: [PATCH 01/10] Added the start of RPC + removed proxy functions. * Set the listener of the server when opened. * refactor server RPC. * deleted proxy functions + moved TCPPeer to tcp file * implemented the start of JSON-RPC * changed string port args to int * added peerCount. * Start a server with startOpts * Updated README --- README.md | 16 ++- cmd/neoserver/main.go | 33 ++++-- pkg/network/peer.go | 80 --------------- pkg/network/rpc.go | 129 ++++++++++++++++++++++++ pkg/network/server.go | 201 +++++++++++++++---------------------- pkg/network/server_test.go | 14 +++ pkg/network/tcp.go | 88 +++++++++++++++- 7 files changed, 341 insertions(+), 220 deletions(-) create mode 100644 pkg/network/rpc.go diff --git a/README.md b/README.md index 1150445c7..87e863e74 100644 --- a/README.md +++ b/README.md @@ -34,9 +34,9 @@ The project will exist out of the following topics/packages: 1. network (started) 2. core (started) -3. vm (open) -4. smartcontract (open) -5. api (RPC server) (open) +3. api (JSON-RPC server) (started) +4. vm (open) +5. smartcontract (open) # Getting started ### Server @@ -53,8 +53,16 @@ You can add multiple seeds if you want: `neoserver -seed 127.0.0.1:20333,127.0.01:20334` +By default the server will currently run on port 3000, for testing purposes. You can change that by setting the tcp flag: + +`neoserver -seed 127.0.0.1:20333 -tcp 1337` + ### RPC -To be implemented.. +If you want your node to also serve JSON-RPC, you can do that by setting the following flag: + +`neoserver -rpc 4000` + +In this case server will accept and respond JSON-RPC on port 4000. Keep in mind that currently there is only a small subset of the JSON-RPC implemented. Feel free to make a PR with more functionality. ### vm To be implemented.. diff --git a/cmd/neoserver/main.go b/cmd/neoserver/main.go index f06c73c3d..fff926922 100644 --- a/cmd/neoserver/main.go +++ b/cmd/neoserver/main.go @@ -8,25 +8,36 @@ import ( ) var ( - port = flag.String("port", ":3000", "port the TCP listener will listen on.") + tcp = flag.Int("tcp", 3000, "port TCP listener will listen on.") seed = flag.String("seed", "", "initial seed servers.") net = flag.Int("net", 56753, "the mode the server will operate in.") + rpc = flag.Int("rpc", 0, "let this server also respond to rpc calls on this port") ) // Simple dirty and quick bootstrapping for the sake of development. // e.g run 2 nodes: -// neoserver -port :4000 -// neoserver -port :3000 -seed 127.0.0.1:4000 +// neoserver -tcp :4000 +// neoserver -tcp :3000 -seed 127.0.0.1:4000 func main() { flag.Parse() + opts := network.StartOpts{ + Seeds: parseSeeds(*seed), + TCP: *tcp, + RPC: *rpc, + } + s := network.NewServer(network.NetMode(*net)) - seeds := strings.Split(*seed, ",") - if len(seeds) == 0 { - seeds = []string{*seed} - } - if *seed == "" { - seeds = []string{} - } - s.Start(*port, seeds) + s.Start(opts) +} + +func parseSeeds(s string) []string { + if len(s) == 0 { + return nil + } + seeds := strings.Split(s, ",") + if len(seeds) == 0 { + return nil + } + return seeds } diff --git a/pkg/network/peer.go b/pkg/network/peer.go index af38fe142..025f4b7d8 100644 --- a/pkg/network/peer.go +++ b/pkg/network/peer.go @@ -1,8 +1,6 @@ package network import ( - "net" - "github.com/anthdm/neo-go/pkg/util" ) @@ -44,81 +42,3 @@ func (p *LocalPeer) id() uint32 { return p.nonce } func (p *LocalPeer) verack() bool { return p.isVerack } func (p *LocalPeer) addr() util.Endpoint { return p.endpoint } func (p *LocalPeer) disconnect() {} - -// TCPPeer represents a remote node, backed by TCP transport. -type TCPPeer struct { - s *Server - // nonce (id) of the peer. - nonce uint32 - // underlying TCP connection - conn net.Conn - // host and port information about this peer. - endpoint util.Endpoint - // channel to coordinate messages writen back to the connection. - send chan *Message - // whether this peers version was acknowledged. - isVerack bool -} - -// NewTCPPeer returns a pointer to a TCP Peer. -func NewTCPPeer(conn net.Conn, s *Server) *TCPPeer { - e, _ := util.EndpointFromString(conn.RemoteAddr().String()) - - return &TCPPeer{ - conn: conn, - send: make(chan *Message), - endpoint: e, - s: s, - } -} - -func (p *TCPPeer) callVersion(msg *Message) { - p.send <- msg -} - -// id implements the peer interface -func (p *TCPPeer) id() uint32 { - return p.nonce -} - -// endpoint implements the peer interface -func (p *TCPPeer) addr() util.Endpoint { - return p.endpoint -} - -// verack implements the peer interface -func (p *TCPPeer) verack() bool { - return p.isVerack -} - -// callGetaddr will send the "getaddr" command to the remote. -func (p *TCPPeer) callGetaddr(msg *Message) { - p.send <- msg -} - -func (p *TCPPeer) disconnect() { - close(p.send) - p.conn.Close() -} - -// writeLoop writes messages to the underlying TCP connection. -// A goroutine writeLoop is started for each connection. -// There should be at most one writer to a connection executing -// all writes from this goroutine. -func (p *TCPPeer) writeLoop() { - // clean up the connection. - defer func() { - p.conn.Close() - }() - - for { - msg := <-p.send - - p.s.logger.Printf("OUT :: %s :: %+v", msg.commandType(), msg.Payload) - - // should we disconnect here? - if err := msg.encode(p.conn); err != nil { - p.s.logger.Printf("encode error: %s", err) - } - } -} diff --git a/pkg/network/rpc.go b/pkg/network/rpc.go new file mode 100644 index 000000000..c1156fa65 --- /dev/null +++ b/pkg/network/rpc.go @@ -0,0 +1,129 @@ +package network + +import ( + "encoding/json" + "fmt" + "net/http" +) + +const ( + rpcPortMainNet = 20332 + rpcPortTestNet = 10332 + rpcVersion = "2.0" + + // error response messages + methodNotFound = "Method not found" + parseError = "Parse error" +) + +// Each NEO node has a set of optional APIs for accessing blockchain +// data and making things easier for development of blockchain apps. +// APIs are provided via JSON-RPC , comm at bottom layer is with http/https protocol. + +// listenHTTP creates an ingress bridge from the outside world to the passed +// server, by installing handlers for all the necessary RPCs to the passed mux. +func listenHTTP(s *Server, port int) { + api := &API{s} + p := fmt.Sprintf(":%d", port) + s.logger.Printf("serving RPC on %d", port) + s.logger.Printf("%s", http.ListenAndServe(p, api)) +} + +// API serves JSON-RPC. +type API struct { + s *Server +} + +func (s *API) ServeHTTP(w http.ResponseWriter, r *http.Request) { + // Official nodes respond a parse error if the method is not POST. + // Instead of returning a decent response for this, let's do the same. + if r.Method != "POST" { + writeError(w, 0, 0, parseError) + } + + var req Request + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + writeError(w, 0, 0, parseError) + return + } + defer r.Body.Close() + + if req.Version != rpcVersion { + writeJSON(w, http.StatusBadRequest, nil) + return + } + + switch req.Method { + case "getconnectioncount": + if err := s.getConnectionCount(w, &req); err != nil { + writeError(w, 0, 0, parseError) + return + } + case "getblockcount": + case "getbestblockhash": + default: + writeError(w, 0, 0, methodNotFound) + } +} + +// This is an Example on how we could handle incomming RPC requests. +func (s *API) getConnectionCount(w http.ResponseWriter, req *Request) error { + count := s.s.peerCount() + + resp := ConnectionCountResponse{ + Version: rpcVersion, + Result: count, + ID: 1, + } + + return writeJSON(w, http.StatusOK, resp) +} + +// writeError returns a JSON error with given parameters. All error HTTP +// status codes are 200. According to the official API. +func writeError(w http.ResponseWriter, id, code int, msg string) error { + resp := RequestError{ + Version: rpcVersion, + ID: id, + Error: Error{ + Code: code, + Message: msg, + }, + } + + return writeJSON(w, http.StatusOK, resp) +} + +func writeJSON(w http.ResponseWriter, status int, v interface{}) error { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(status) + return json.NewEncoder(w).Encode(v) +} + +// Request is an object received through JSON-RPC from the client. +type Request struct { + Version string `json:"jsonrpc"` + Method string `json:"method"` + Params []string `json:"params"` + ID int `json:"id"` +} + +// ConnectionCountResponse .. +type ConnectionCountResponse struct { + Version string `json:"jsonrpc"` + Result int `json:"result"` + ID int `json:"id"` +} + +// RequestError .. +type RequestError struct { + Version string `json:"jsonrpc"` + ID int `json:"id"` + Error Error `json:"error"` +} + +// Error holds information about an RCP error. +type Error struct { + Code int `json:"code"` + Message string `json:"message"` +} diff --git a/pkg/network/server.go b/pkg/network/server.go index 36dd70fa9..c18c11695 100644 --- a/pkg/network/server.go +++ b/pkg/network/server.go @@ -5,7 +5,6 @@ import ( "log" "net" "os" - "strconv" "time" "github.com/anthdm/neo-go/pkg/network/payload" @@ -53,12 +52,8 @@ type Server struct { relay bool // TCP listener of the server listener net.Listener - - // RPC channels - versionCh chan versionTuple - getaddrCh chan getaddrTuple - invCh chan invTuple - addrCh chan addrTuple + // channel for safely responding the number of current connected peers. + peerCountCh chan peerCount } // NewServer returns a pointer to a new server. @@ -70,32 +65,26 @@ func NewServer(net NetMode) *Server { } s := &Server{ - id: util.RandUint32(1111111, 9999999), - userAgent: fmt.Sprintf("/NEO:%s/", version), - logger: logger, - peers: make(map[Peer]bool), - register: make(chan Peer), - unregister: make(chan Peer), - message: make(chan messageTuple), - relay: true, // currently relay is not handled. - net: net, - quit: make(chan struct{}), - versionCh: make(chan versionTuple), - getaddrCh: make(chan getaddrTuple), - invCh: make(chan invTuple), - addrCh: make(chan addrTuple), + id: util.RandUint32(1111111, 9999999), + userAgent: fmt.Sprintf("/NEO:%s/", version), + logger: logger, + peers: make(map[Peer]bool), + register: make(chan Peer), + unregister: make(chan Peer), + message: make(chan messageTuple), + relay: true, // currently relay is not handled. + net: net, + quit: make(chan struct{}), + peerCountCh: make(chan peerCount), } return s } // Start run's the server. -func (s *Server) Start(port string, seeds []string) { - p, err := strconv.Atoi(port[1:len(port)]) - if err != nil { - s.logger.Fatalf("could not convert port to integer: %s", err) - } - s.port = uint16(p) +// TODO: server should be initialized with a config. +func (s *Server) Start(opts StartOpts) { + s.port = uint16(opts.TCP) fmt.Println(logo()) fmt.Println(string(s.userAgent)) @@ -103,10 +92,14 @@ func (s *Server) Start(port string, seeds []string) { s.logger.Printf("NET: %s - TCP: %d - RELAY: %v - ID: %d", s.net, int(s.port), s.relay, s.id) - go listenTCP(s, port) + go listenTCP(s, opts.TCP) - if len(seeds) > 0 { - connectToSeeds(s, seeds) + if opts.RPC > 0 { + go listenHTTP(s, opts.RPC) + } + + if len(opts.Seeds) > 0 { + connectToSeeds(s, opts.Seeds) } s.loop() @@ -147,44 +140,8 @@ func (s *Server) loop() { s.logger.Printf("peer %s disconnected", peer.addr()) } - // Process the received version and respond with a verack. - case t := <-s.versionCh: - if s.id == t.request.Nonce { - t.peer.disconnect() - } - if t.peer.addr().Port != t.request.Port { - t.peer.disconnect() - } - t.response <- newMessage(ModeDevNet, cmdVerack, nil) - - // Process the getaddr cmd. - case t := <-s.getaddrCh: - t.response <- &Message{} // just for now. - - // Process the addr cmd. Register peer will handle the maxPeers connected. - case t := <-s.addrCh: - for _, addr := range t.request.Addrs { - if !s.peerAlreadyConnected(addr.Addr) { - // TODO: this is not transport abstracted. - go connectToRemoteNode(s, addr.Addr.String()) - } - } - t.response <- true - - // Process inventories cmd. - case t := <-s.invCh: - if !t.request.Type.Valid() { - t.peer.disconnect() - break - } - if len(t.request.Hashes) == 0 { - t.peer.disconnect() - break - } - - payload := payload.NewInventory(t.request.Type, t.request.Hashes) - msg := newMessage(s.net, cmdGetData, payload) - t.response <- msg + case t := <-s.peerCountCh: + t.count <- len(s.peers) case <-s.quit: s.shutdown() @@ -202,73 +159,47 @@ func (s *Server) handlePeerConnected(p Peer) { p.callVersion(msg) } -type versionTuple struct { - peer Peer - request *payload.Version - response chan *Message -} - func (s *Server) handleVersionCmd(msg *Message, p Peer) *Message { - t := versionTuple{ - peer: p, - request: msg.Payload.(*payload.Version), - response: make(chan *Message), + version := msg.Payload.(*payload.Version) + if s.id == version.Nonce { + p.disconnect() + return nil } - - s.versionCh <- t - - return <-t.response -} - -type getaddrTuple struct { - peer Peer - request *Message - response chan *Message + if p.addr().Port != version.Port { + p.disconnect() + return nil + } + return newMessage(ModeDevNet, cmdVerack, nil) } func (s *Server) handleGetaddrCmd(msg *Message, p Peer) *Message { - t := getaddrTuple{ - peer: p, - request: msg, - response: make(chan *Message), - } - - s.getaddrCh <- t - - return <-t.response -} - -type invTuple struct { - peer Peer - request *payload.Inventory - response chan *Message + return nil } func (s *Server) handleInvCmd(msg *Message, p Peer) *Message { - t := invTuple{ - request: msg.Payload.(*payload.Inventory), - response: make(chan *Message), + inv := msg.Payload.(*payload.Inventory) + if !inv.Type.Valid() { + p.disconnect() + return nil + } + if len(inv.Hashes) == 0 { + p.disconnect() + return nil } - s.invCh <- t - - return <-t.response + payload := payload.NewInventory(inv.Type, inv.Hashes) + resp := newMessage(s.net, cmdGetData, payload) + return resp } -type addrTuple struct { - request *payload.AddressList - response chan bool -} - -func (s *Server) handleAddrCmd(msg *Message, p Peer) bool { - t := addrTuple{ - request: msg.Payload.(*payload.AddressList), - response: make(chan bool), +func (s *Server) handleAddrCmd(msg *Message, p Peer) { + addrList := msg.Payload.(*payload.AddressList) + for _, addr := range addrList.Addrs { + if !s.peerAlreadyConnected(addr.Addr) { + // TODO: this is not transport abstracted. + go connectToRemoteNode(s, addr.Addr.String()) + } } - - s.addrCh <- t - - return <-t.response } // check if the addr is already connected to the server. @@ -283,6 +214,7 @@ func (s *Server) peerAlreadyConnected(addr net.Addr) bool { func (s *Server) sendLoop(peer Peer) { // TODO: check if this peer is still connected. + // dont keep asking (maxPeers and no new nodes) for { getaddrMsg := newMessage(s.net, cmdGetAddr, nil) peer.callGetaddr(getaddrMsg) @@ -291,6 +223,31 @@ func (s *Server) sendLoop(peer Peer) { } } +type peerCount struct { + count chan int +} + +// peerCount returns the number of connected peers to this server. +func (s *Server) peerCount() int { + ch := peerCount{ + count: make(chan int), + } + + s.peerCountCh <- ch + + return <-ch.count +} + +// StartOpts holds the server configuration. +type StartOpts struct { + // tcp port + TCP int + // slice of peer addresses the server will connect to + Seeds []string + // JSON-RPC port. If 0 no RPC handler will be attached. + RPC int +} + func logo() string { return ` _ ____________ __________ diff --git a/pkg/network/server_test.go b/pkg/network/server_test.go index ad07d93e1..f0ba721f7 100644 --- a/pkg/network/server_test.go +++ b/pkg/network/server_test.go @@ -24,6 +24,20 @@ func TestHandleVersion(t *testing.T) { } } +func TestPeerCount(t *testing.T) { + s := NewServer(ModeDevNet) + go s.loop() + + lenPeers := 10 + for i := 0; i < lenPeers; i++ { + s.register <- NewLocalPeer(s) + } + + if have, want := s.peerCount(), lenPeers; want != have { + t.Fatalf("expected %d connected peers got %d", want, have) + } +} + func TestHandleAddrCmd(t *testing.T) { // todo } diff --git a/pkg/network/tcp.go b/pkg/network/tcp.go index f9ffc39e0..a1280a6b0 100644 --- a/pkg/network/tcp.go +++ b/pkg/network/tcp.go @@ -2,13 +2,15 @@ package network import ( "bytes" + "fmt" "net" "github.com/anthdm/neo-go/pkg/network/payload" + "github.com/anthdm/neo-go/pkg/util" ) -func listenTCP(s *Server, port string) error { - ln, err := net.Listen("tcp", port) +func listenTCP(s *Server, port int) error { + ln, err := net.Listen("tcp", fmt.Sprintf(":%d", port)) if err != nil { return err } @@ -73,10 +75,11 @@ func handleConnection(s *Server, conn net.Conn) { } } +// handleMessage hands the message received from a TCP connection over to the server. func handleMessage(msg *Message, s *Server, p *TCPPeer) { command := msg.commandType() - s.logger.Printf("%d :: IN :: %s :: %v", p.id(), command, msg) + s.logger.Printf("IN :: %d :: %s :: %v", p.id(), command, msg) switch command { case cmdVersion: @@ -103,3 +106,82 @@ func handleMessage(msg *Message, s *Server, p *TCPPeer) { default: } } + +// TCPPeer represents a remote node, backed by TCP transport. +type TCPPeer struct { + s *Server + // nonce (id) of the peer. + nonce uint32 + // underlying TCP connection + conn net.Conn + // host and port information about this peer. + endpoint util.Endpoint + // channel to coordinate messages writen back to the connection. + send chan *Message + // whether this peers version was acknowledged. + isVerack bool +} + +// NewTCPPeer returns a pointer to a TCP Peer. +func NewTCPPeer(conn net.Conn, s *Server) *TCPPeer { + e, _ := util.EndpointFromString(conn.RemoteAddr().String()) + + return &TCPPeer{ + conn: conn, + send: make(chan *Message), + endpoint: e, + s: s, + } +} + +func (p *TCPPeer) callVersion(msg *Message) { + p.send <- msg +} + +// id implements the peer interface +func (p *TCPPeer) id() uint32 { + return p.nonce +} + +// endpoint implements the peer interface +func (p *TCPPeer) addr() util.Endpoint { + return p.endpoint +} + +// verack implements the peer interface +func (p *TCPPeer) verack() bool { + return p.isVerack +} + +// callGetaddr will send the "getaddr" command to the remote. +func (p *TCPPeer) callGetaddr(msg *Message) { + p.send <- msg +} + +// disconnect closes the send channel and the underlying connection. +func (p *TCPPeer) disconnect() { + close(p.send) + p.conn.Close() +} + +// writeLoop writes messages to the underlying TCP connection. +// A goroutine writeLoop is started for each connection. +// There should be at most one writer to a connection executing +// all writes from this goroutine. +func (p *TCPPeer) writeLoop() { + // clean up the connection. + defer func() { + p.conn.Close() + }() + + for { + msg := <-p.send + + p.s.logger.Printf("OUT :: %s :: %+v", msg.commandType(), msg.Payload) + + // should we disconnect here? + if err := msg.encode(p.conn); err != nil { + p.s.logger.Printf("encode error: %s", err) + } + } +} From 04e9060484d28c96d5937569869ab471b9e6f342 Mon Sep 17 00:00:00 2001 From: anthdm Date: Thu, 1 Feb 2018 10:25:34 +0100 Subject: [PATCH 02/10] Added GetBlocks payload --- pkg/network/payload/getblocks.go | 48 +++++++++++++++++++++++++++ pkg/network/payload/getblocks_test.go | 37 +++++++++++++++++++++ 2 files changed, 85 insertions(+) create mode 100644 pkg/network/payload/getblocks.go create mode 100644 pkg/network/payload/getblocks_test.go diff --git a/pkg/network/payload/getblocks.go b/pkg/network/payload/getblocks.go new file mode 100644 index 000000000..909335b9a --- /dev/null +++ b/pkg/network/payload/getblocks.go @@ -0,0 +1,48 @@ +package payload + +import ( + "encoding/binary" + "io" + + . "github.com/anthdm/neo-go/pkg/util" +) + +// GetBlocks payload +type GetBlocks struct { + // hash of latest block that node requests + HashStart []Uint256 + // hash of last block that node requests + HashStop Uint256 +} + +// NewGetBlocks return a pointer to a GetBlocks object. +func NewGetBlocks(start []Uint256, stop Uint256) *GetBlocks { + return &GetBlocks{ + HashStart: start, + HashStop: stop, + } +} + +// DecodeBinary implements the payload interface. +func (p *GetBlocks) DecodeBinary(r io.Reader) error { + var lenStart uint8 + + err := binary.Read(r, binary.LittleEndian, &lenStart) + p.HashStart = make([]Uint256, lenStart) + err = binary.Read(r, binary.LittleEndian, &p.HashStart) + err = binary.Read(r, binary.LittleEndian, &p.HashStop) + + return err +} + +// EncodeBinary implements the payload interface. +func (p *GetBlocks) EncodeBinary(w io.Writer) error { + err := binary.Write(w, binary.LittleEndian, uint8(len(p.HashStart))) + err = binary.Write(w, binary.LittleEndian, p.HashStart) + err = binary.Write(w, binary.LittleEndian, p.HashStop) + + return err +} + +// Size implements the payload interface. +func (p *GetBlocks) Size() uint32 { return 0 } diff --git a/pkg/network/payload/getblocks_test.go b/pkg/network/payload/getblocks_test.go new file mode 100644 index 000000000..c73772386 --- /dev/null +++ b/pkg/network/payload/getblocks_test.go @@ -0,0 +1,37 @@ +package payload + +import ( + "bytes" + "crypto/sha256" + "reflect" + "testing" + + . "github.com/anthdm/neo-go/pkg/util" +) + +func TestGetBlocksEncodeDecode(t *testing.T) { + start := []Uint256{ + sha256.Sum256([]byte("a")), + sha256.Sum256([]byte("b")), + } + stop := sha256.Sum256([]byte("c")) + + p := NewGetBlocks(start, stop) + buf := new(bytes.Buffer) + if err := p.EncodeBinary(buf); err != nil { + t.Fatal(err) + } + + if have, want := buf.Len(), 1+64+32; have != want { + t.Fatalf("expecting a length of %d got %d", want, have) + } + + pDecode := &GetBlocks{} + if err := pDecode.DecodeBinary(buf); err != nil { + t.Fatal(err) + } + + if !reflect.DeepEqual(p, pDecode) { + t.Fatalf("expecting both getblocks payloads to be equal %v and %v", p, pDecode) + } +} From 63072ebe75cd2ebb1c58d0f23a393bc3cb787b7b Mon Sep 17 00:00:00 2001 From: anthdm Date: Thu, 1 Feb 2018 10:56:33 +0100 Subject: [PATCH 03/10] Added getheaders payload + abstracted the fields. --- pkg/network/payload/getblocks.go | 32 +++++++++++++--------- pkg/network/payload/getheaders.go | 17 ++++++++++++ pkg/network/payload/getheaders_test.go | 37 ++++++++++++++++++++++++++ 3 files changed, 73 insertions(+), 13 deletions(-) create mode 100644 pkg/network/payload/getheaders.go create mode 100644 pkg/network/payload/getheaders_test.go diff --git a/pkg/network/payload/getblocks.go b/pkg/network/payload/getblocks.go index 909335b9a..81371cb25 100644 --- a/pkg/network/payload/getblocks.go +++ b/pkg/network/payload/getblocks.go @@ -7,24 +7,17 @@ import ( . "github.com/anthdm/neo-go/pkg/util" ) -// GetBlocks payload -type GetBlocks struct { +// HashStartStop contains fields and methods to be shared with the +// "GetBlocks" and "GetHeaders" payload. +type HashStartStop struct { // hash of latest block that node requests HashStart []Uint256 // hash of last block that node requests HashStop Uint256 } -// NewGetBlocks return a pointer to a GetBlocks object. -func NewGetBlocks(start []Uint256, stop Uint256) *GetBlocks { - return &GetBlocks{ - HashStart: start, - HashStop: stop, - } -} - // DecodeBinary implements the payload interface. -func (p *GetBlocks) DecodeBinary(r io.Reader) error { +func (p *HashStartStop) DecodeBinary(r io.Reader) error { var lenStart uint8 err := binary.Read(r, binary.LittleEndian, &lenStart) @@ -36,7 +29,7 @@ func (p *GetBlocks) DecodeBinary(r io.Reader) error { } // EncodeBinary implements the payload interface. -func (p *GetBlocks) EncodeBinary(w io.Writer) error { +func (p *HashStartStop) EncodeBinary(w io.Writer) error { err := binary.Write(w, binary.LittleEndian, uint8(len(p.HashStart))) err = binary.Write(w, binary.LittleEndian, p.HashStart) err = binary.Write(w, binary.LittleEndian, p.HashStop) @@ -45,4 +38,17 @@ func (p *GetBlocks) EncodeBinary(w io.Writer) error { } // Size implements the payload interface. -func (p *GetBlocks) Size() uint32 { return 0 } +func (p *HashStartStop) Size() uint32 { return 0 } + +// GetBlocks payload +type GetBlocks struct { + HashStartStop +} + +// NewGetBlocks return a pointer to a GetBlocks object. +func NewGetBlocks(start []Uint256, stop Uint256) *GetBlocks { + p := &GetBlocks{} + p.HashStart = start + p.HashStop = stop + return p +} diff --git a/pkg/network/payload/getheaders.go b/pkg/network/payload/getheaders.go new file mode 100644 index 000000000..9ade274f7 --- /dev/null +++ b/pkg/network/payload/getheaders.go @@ -0,0 +1,17 @@ +package payload + +import "github.com/anthdm/neo-go/pkg/util" + +// GetHeaders payload is the same as the "GetBlocks" payload. +type GetHeaders struct { + HashStartStop +} + +// NewGetHeaders return a pointer to a GetHeaders object. +func NewGetHeaders(start []util.Uint256, stop util.Uint256) *GetHeaders { + p := &GetHeaders{} + p.HashStart = start + p.HashStop = stop + + return p +} diff --git a/pkg/network/payload/getheaders_test.go b/pkg/network/payload/getheaders_test.go new file mode 100644 index 000000000..32cbb1b86 --- /dev/null +++ b/pkg/network/payload/getheaders_test.go @@ -0,0 +1,37 @@ +package payload + +import ( + "bytes" + "crypto/sha256" + "reflect" + "testing" + + "github.com/anthdm/neo-go/pkg/util" +) + +func TestGetHeadersEncodeDecode(t *testing.T) { + start := []util.Uint256{ + sha256.Sum256([]byte("a")), + sha256.Sum256([]byte("b")), + } + stop := sha256.Sum256([]byte("c")) + + p := NewGetHeaders(start, stop) + buf := new(bytes.Buffer) + if err := p.EncodeBinary(buf); err != nil { + t.Fatal(err) + } + + if have, want := buf.Len(), 1+64+32; have != want { + t.Fatalf("expecting a length of %d got %d", want, have) + } + + pDecode := &GetHeaders{} + if err := pDecode.DecodeBinary(buf); err != nil { + t.Fatal(err) + } + + if !reflect.DeepEqual(p, pDecode) { + t.Fatalf("expecting both getheaders payloads to be equal %v and %v", p, pDecode) + } +} From b416a51db7ebb94f4b5540039a5f7bf2140921e2 Mon Sep 17 00:00:00 2001 From: anthdm Date: Thu, 1 Feb 2018 14:53:49 +0100 Subject: [PATCH 04/10] tweaked TCP transport + finished version + verack. --- pkg/network/peer.go | 3 -- pkg/network/tcp.go | 85 ++++++++++++++++++++++++++------------------- 2 files changed, 50 insertions(+), 38 deletions(-) diff --git a/pkg/network/peer.go b/pkg/network/peer.go index 025f4b7d8..36a4f87f2 100644 --- a/pkg/network/peer.go +++ b/pkg/network/peer.go @@ -9,7 +9,6 @@ import ( type Peer interface { id() uint32 addr() util.Endpoint - verack() bool disconnect() callVersion(*Message) callGetaddr(*Message) @@ -20,7 +19,6 @@ type Peer interface { type LocalPeer struct { s *Server nonce uint32 - isVerack bool endpoint util.Endpoint } @@ -39,6 +37,5 @@ func (p *LocalPeer) callGetaddr(msg *Message) { } func (p *LocalPeer) id() uint32 { return p.nonce } -func (p *LocalPeer) verack() bool { return p.isVerack } func (p *LocalPeer) addr() util.Endpoint { return p.endpoint } func (p *LocalPeer) disconnect() {} diff --git a/pkg/network/tcp.go b/pkg/network/tcp.go index a1280a6b0..648fe6274 100644 --- a/pkg/network/tcp.go +++ b/pkg/network/tcp.go @@ -22,6 +22,7 @@ func listenTCP(s *Server, port int) error { if err != nil { return err } + go handleConnection(s, conn) } } @@ -54,8 +55,10 @@ func handleConnection(s *Server, conn net.Conn) { s.unregister <- peer }() - // Start a goroutine that will handle all writes to the registered peer. + // Start a goroutine that will handle all outgoing messages. go peer.writeLoop() + // Start a goroutine that will handle all incomming messages. + go handleMessage(s, peer) // Read from the connection and decode it into a Message ready for processing. buf := make([]byte, 1024) @@ -71,39 +74,55 @@ func handleConnection(s *Server, conn net.Conn) { s.logger.Printf("decode error %s", err) break } - handleMessage(msg, s, peer) + + peer.receive <- msg } } // handleMessage hands the message received from a TCP connection over to the server. -func handleMessage(msg *Message, s *Server, p *TCPPeer) { - command := msg.commandType() +func handleMessage(s *Server, p *TCPPeer) { + // Disconnect the peer when we break out of the loop. + defer func() { + p.disconnect() + }() - s.logger.Printf("IN :: %d :: %s :: %v", p.id(), command, msg) + for { + msg := <-p.receive + command := msg.commandType() - switch command { - case cmdVersion: - resp := s.handleVersionCmd(msg, p) - p.isVerack = true - p.nonce = msg.Payload.(*payload.Version).Nonce - p.send <- resp - case cmdAddr: - s.handleAddrCmd(msg, p) - case cmdGetAddr: - s.handleGetaddrCmd(msg, p) - case cmdInv: - resp := s.handleInvCmd(msg, p) - p.send <- resp - case cmdBlock: - case cmdConsensus: - case cmdTX: - case cmdVerack: - go s.sendLoop(p) - case cmdGetHeaders: - case cmdGetBlocks: - case cmdGetData: - case cmdHeaders: - default: + s.logger.Printf("IN :: %d :: %s :: %v", p.id(), command, msg) + + switch command { + case cmdVersion: + resp := s.handleVersionCmd(msg, p) + p.nonce = msg.Payload.(*payload.Version).Nonce + p.send <- resp + + // after sending our version we want a "verack" and nothing else. + msg := <-p.receive + if msg.commandType() != cmdVerack { + break + } + // we can start the protocol now. + go s.sendLoop(p) + case cmdAddr: + s.handleAddrCmd(msg, p) + case cmdGetAddr: + s.handleGetaddrCmd(msg, p) + case cmdInv: + resp := s.handleInvCmd(msg, p) + p.send <- resp + case cmdBlock: + case cmdConsensus: + case cmdTX: + case cmdVerack: + // disconnect the peer, verack should already be handled. + break + case cmdGetHeaders: + case cmdGetBlocks: + case cmdGetData: + case cmdHeaders: + } } } @@ -118,8 +137,8 @@ type TCPPeer struct { endpoint util.Endpoint // channel to coordinate messages writen back to the connection. send chan *Message - // whether this peers version was acknowledged. - isVerack bool + // channel to receive from underlying connection. + receive chan *Message } // NewTCPPeer returns a pointer to a TCP Peer. @@ -129,6 +148,7 @@ func NewTCPPeer(conn net.Conn, s *Server) *TCPPeer { return &TCPPeer{ conn: conn, send: make(chan *Message), + receive: make(chan *Message), endpoint: e, s: s, } @@ -148,11 +168,6 @@ func (p *TCPPeer) addr() util.Endpoint { return p.endpoint } -// verack implements the peer interface -func (p *TCPPeer) verack() bool { - return p.isVerack -} - // callGetaddr will send the "getaddr" command to the remote. func (p *TCPPeer) callGetaddr(msg *Message) { p.send <- msg From 5b9578db5d930701d0dcef104ccebe3355edf0dd Mon Sep 17 00:00:00 2001 From: Charlie Revett Date: Thu, 1 Feb 2018 09:40:04 -0800 Subject: [PATCH 05/10] Repo Setup (#6) --- .gitignore | 17 +++++++++ Makefile | 2 ++ README.md | 103 ++++++++++++++++++++++++++++++++++++----------------- glide.lock | 11 ++++++ glide.yaml | 8 +++++ 5 files changed, 109 insertions(+), 32 deletions(-) create mode 100644 .gitignore create mode 100644 Makefile create mode 100644 glide.lock create mode 100644 glide.yaml diff --git a/.gitignore b/.gitignore new file mode 100644 index 000000000..292589f8f --- /dev/null +++ b/.gitignore @@ -0,0 +1,17 @@ +# Binaries for programs and plugins +*.exe +*.dll +*.so +*.dylib + +# Test binary, build with `go test -c` +*.test + +# Output of the go coverage tool, specifically when used with LiteIDE +*.out + +# Project-local glide cache, RE: https://github.com/Masterminds/glide/issues/736 +.glide/ + +# Added by CoZ developers +vendor/ \ No newline at end of file diff --git a/Makefile b/Makefile new file mode 100644 index 000000000..18a8f2e72 --- /dev/null +++ b/Makefile @@ -0,0 +1,2 @@ +test: + @go test $(glide nv) -cover \ No newline at end of file diff --git a/README.md b/README.md index 87e863e74..dda08b6ba 100644 --- a/README.md +++ b/README.md @@ -5,10 +5,10 @@ >

-

NEO-GO

+

neo-go

- Node and SDK for the NEO blockchain written in the Go language. + Go Node and SDK for the NEO blockchain.

@@ -18,67 +18,106 @@

# Overview -> This project is currently in alpha and under active development. -### Long term project goals -Full port of the original C# [NEO project](https://github.com/neo-project). A complete toolkit for the NEO blockchain. +> This project is currently in **alpha** and under active development. -- Full server (consensus and RPC) nodes. +## Project Goals + +Full port of the original C# [NEO project](https://github.com/neo-project). +A complete toolkit for the NEO blockchain, including: + +- Full consensus node +- Full RPC node - RPC client -- build, compile and deploy smart contracts with the Go vm +- ClI tool +- Smart contract compiler -### Current state -This project is still under heavy development. Still working on internal API's and project layout. This should not take longer than 2 weeks. +## Current State -The project will exist out of the following topics/packages: +This project is still under heavy development. Still working on internal API's and project layout. T +his should not take longer than 2 weeks. -1. network (started) -2. core (started) -3. api (JSON-RPC server) (started) -4. vm (open) -5. smartcontract (open) +The project will exist out of the following packages: -# Getting started -### Server +| Package | State | Developer | +|---------------|---------|--------------------------------------| +| api | started | [@anthdm](https://github.com/anthdm) | +| core | started | [@anthdm](https://github.com/anthdm) | +| network | started | [@anthdm](https://github.com/anthdm) | +| smartcontract | started | [@revett](https://github.com/revett) | +| vm | started | [@revett](https://github.com/revett) | -Install the neoserver cli `go install ./cmd/neoserver` +# Getting Started -Currently, there is a minimal subset of the NEO protocol implemented. To start experimenting make sure you a have a private net running on your machine. If you dont, take a look at [docker-privnet-with-gas](https://hub.docker.com/r/metachris/neo-privnet-with-gas/). +## Server + +Install the neoserver cli: + + ``` + go install ./cmd/neoserver + ``` + +Currently, there is a minimal subset of the NEO protocol implemented. +To start experimenting make sure you a have a private net running on your machine. +If you dont, take a look at [docker-privnet-with-gas](https://hub.docker.com/r/metachris/neo-privnet-with-gas/). Start the server: -`neoserver -seed 127.0.0.1:20333` +``` +neoserver -seed 127.0.0.1:20333 +``` You can add multiple seeds if you want: -`neoserver -seed 127.0.0.1:20333,127.0.01:20334` +``` +neoserver -seed 127.0.0.1:20333,127.0.01:20334 +``` -By default the server will currently run on port 3000, for testing purposes. You can change that by setting the tcp flag: +By default the server will currently run on port 3000, for testing purposes. +You can change that by setting the tcp flag: -`neoserver -seed 127.0.0.1:20333 -tcp 1337` +``` +neoserver -seed 127.0.0.1:20333 -tcp 1337 +``` + +## RPC -### RPC If you want your node to also serve JSON-RPC, you can do that by setting the following flag: -`neoserver -rpc 4000` +``` +neoserver -rpc 4000 +``` -In this case server will accept and respond JSON-RPC on port 4000. Keep in mind that currently there is only a small subset of the JSON-RPC implemented. Feel free to make a PR with more functionality. +In this case server will accept and respond JSON-RPC on port 4000. +Keep in mind that currently there is only a small subset of the JSON-RPC implemented. +Feel free to make a PR with more functionality. -### vm -To be implemented.. +## VM -### smart contracts -To be implemented.. +``` +TODO +``` + +## Smart Contracts + +``` +TODO +``` # Contributing -Feel free to contribute to this project after reading the [contributing guidelines](https://github.com/anthdm/neo-go/blob/master/CONTRIBUTING.md). -Before starting to work on a certain topic, create an new issue first, describing the feauture/topic you are going to implement. +Feel free to contribute to this project after reading the +[contributing guidelines](https://github.com/anthdm/neo-go/blob/master/CONTRIBUTING.md). + +Before starting to work on a certain topic, create an new issue first, +describing the feauture/topic you are going to implement. # Contact + - [@anthdm](https://github.com/anthdm) on Github - [@anthdm](https://twitter.com/anthdm) on Twitter - Send me an email anthony@cityofzion.io # License + - Open-source [MIT](https://github.com/anthdm/neo-go/blob/master/LICENCE.md) diff --git a/glide.lock b/glide.lock new file mode 100644 index 000000000..a3815ad55 --- /dev/null +++ b/glide.lock @@ -0,0 +1,11 @@ +hash: 054e4119c1d6deac9c76a3f6ecc319c80d2099d6dd4fd804b03ce47a9b1ccc86 +updated: 2018-02-01T17:18:00.958758Z +imports: +- name: github.com/anthdm/neo-go + version: 3b91a4808e67c54687a4ba90ac4caf584398cea4 + subpackages: + - pkg/core + - pkg/network + - pkg/network/payload + - pkg/util +testImports: [] diff --git a/glide.yaml b/glide.yaml new file mode 100644 index 000000000..dc88f8824 --- /dev/null +++ b/glide.yaml @@ -0,0 +1,8 @@ +package: github.com/CityOfZion/neo-go +import: +- package: github.com/anthdm/neo-go + subpackages: + - pkg/core + - pkg/network + - pkg/network/payload + - pkg/util From a95ce31176e47320cdbb4c97a7a039da31989f84 Mon Sep 17 00:00:00 2001 From: Charlie Revett Date: Thu, 1 Feb 2018 10:05:56 -0800 Subject: [PATCH 06/10] Separate TransactionType to new file (#8) --- pkg/core/transaction.go | 37 +++++------------------------------- pkg/core/transaction_type.go | 28 +++++++++++++++++++++++++++ 2 files changed, 33 insertions(+), 32 deletions(-) create mode 100644 pkg/core/transaction_type.go diff --git a/pkg/core/transaction.go b/pkg/core/transaction.go index da582461f..bc432bca1 100644 --- a/pkg/core/transaction.go +++ b/pkg/core/transaction.go @@ -5,31 +5,9 @@ import ( "io" ) -// TransactionType is the type of a transaction. -type TransactionType uint8 - -// String implements the stringer interface. -func (t TransactionType) String() string { - switch t { - case MinerTX: - return "miner transaction" - case IssueTX: - return "issue transaction" - case ClaimTX: - return "claim transaction" - case EnrollmentTX: - return "enrollment transaction" - case VotingTX: - return "voting transaction" - case RegisterTX: - return "register transaction" - case ContractTX: - return "contract transaction" - case AgencyTX: - return "agency transaction" - default: - return "" - } +// Transaction is a process recorded in the NEO system. +type Transaction struct { + Type TransactionType } // All processes in NEO system are recorded in transactions. @@ -45,18 +23,13 @@ const ( AgencyTX = 0xb0 ) -// Transaction is a process recorded in the NEO system. -type Transaction struct { - Type TransactionType -} - // DecodeBinary implements the payload interface. -func (t *Transaction) DecodeBinary(r io.Reader) error { +func (t Transaction) DecodeBinary(r io.Reader) error { err := binary.Read(r, binary.LittleEndian, &t.Type) return err } // EncodeBinary implements the payload interface. -func (t *Transaction) EncodeBinary(w io.Writer) error { +func (t Transaction) EncodeBinary(w io.Writer) error { return nil } diff --git a/pkg/core/transaction_type.go b/pkg/core/transaction_type.go new file mode 100644 index 000000000..3aafe0166 --- /dev/null +++ b/pkg/core/transaction_type.go @@ -0,0 +1,28 @@ +package core + +// TransactionType is the type of a transaction. +type TransactionType uint8 + +// String implements the stringer interface. +func (t TransactionType) String() string { + switch t { + case MinerTX: + return "miner transaction" + case IssueTX: + return "issue transaction" + case ClaimTX: + return "claim transaction" + case EnrollmentTX: + return "enrollment transaction" + case VotingTX: + return "voting transaction" + case RegisterTX: + return "register transaction" + case ContractTX: + return "contract transaction" + case AgencyTX: + return "agency transaction" + default: + return "" + } +} From 36335e587ff8a25b0d13e8e44d19241010973fbf Mon Sep 17 00:00:00 2001 From: Charlie Revett Date: Thu, 1 Feb 2018 10:06:17 -0800 Subject: [PATCH 07/10] Simplify CLI build process (#7) --- .gitignore | 3 ++- Makefile | 6 ++++++ README.md | 22 ++++++++++++++-------- {cmd/neoserver => cli}/main.go | 0 4 files changed, 22 insertions(+), 9 deletions(-) rename {cmd/neoserver => cli}/main.go (100%) diff --git a/.gitignore b/.gitignore index 292589f8f..ec197f14d 100644 --- a/.gitignore +++ b/.gitignore @@ -14,4 +14,5 @@ .glide/ # Added by CoZ developers -vendor/ \ No newline at end of file +vendor/ +bin/ \ No newline at end of file diff --git a/Makefile b/Makefile index 18a8f2e72..d4a442545 100644 --- a/Makefile +++ b/Makefile @@ -1,2 +1,8 @@ +build: + @go build -o ./bin/neo-go ./cli/main.go + +deps: + @glide install + test: @go test $(glide nv) -cover \ No newline at end of file diff --git a/README.md b/README.md index dda08b6ba..6aa92a746 100644 --- a/README.md +++ b/README.md @@ -51,11 +51,17 @@ The project will exist out of the following packages: ## Server -Install the neoserver cli: +Install dependencies, this requires [Glide](https://github.com/Masterminds/glide#install): - ``` - go install ./cmd/neoserver - ``` +``` +make deps +``` + +Build the **neo-go** CLI: + +``` +make build +``` Currently, there is a minimal subset of the NEO protocol implemented. To start experimenting make sure you a have a private net running on your machine. @@ -64,20 +70,20 @@ If you dont, take a look at [docker-privnet-with-gas](https://hub.docker.com/r/m Start the server: ``` -neoserver -seed 127.0.0.1:20333 +./bin/neo-go -seed 127.0.0.1:20333 ``` You can add multiple seeds if you want: ``` -neoserver -seed 127.0.0.1:20333,127.0.01:20334 +./bin/neo-go -seed 127.0.0.1:20333,127.0.01:20334 ``` By default the server will currently run on port 3000, for testing purposes. You can change that by setting the tcp flag: ``` -neoserver -seed 127.0.0.1:20333 -tcp 1337 +./bin/neo-go -seed 127.0.0.1:20333 -tcp 1337 ``` ## RPC @@ -85,7 +91,7 @@ neoserver -seed 127.0.0.1:20333 -tcp 1337 If you want your node to also serve JSON-RPC, you can do that by setting the following flag: ``` -neoserver -rpc 4000 +./bin/neo-go -rpc 4000 ``` In this case server will accept and respond JSON-RPC on port 4000. diff --git a/cmd/neoserver/main.go b/cli/main.go similarity index 100% rename from cmd/neoserver/main.go rename to cli/main.go From 0032efcc3b407fbb1ea1a57a8a986e537294dd6f Mon Sep 17 00:00:00 2001 From: Charlie Revett Date: Thu, 1 Feb 2018 18:08:54 +0000 Subject: [PATCH 08/10] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 6aa92a746..0da105a96 100644 --- a/README.md +++ b/README.md @@ -29,7 +29,7 @@ A complete toolkit for the NEO blockchain, including: - Full consensus node - Full RPC node - RPC client -- ClI tool +- CLI tool - Smart contract compiler ## Current State From dd94086a229d61c92124fca34d0eb24a4ebd8930 Mon Sep 17 00:00:00 2001 From: Charlie Revett Date: Thu, 1 Feb 2018 10:54:23 -0800 Subject: [PATCH 09/10] CircleCI 2 & Releases (#9) --- Makefile | 17 ++++- VERSION | 1 + circle.yml | 86 ++++++++++++++++++++++++++ cli/main.go | 2 +- glide.lock | 13 +--- glide.yaml | 8 +-- pkg/core/block.go | 2 +- pkg/network/message.go | 4 +- pkg/network/message_test.go | 2 +- pkg/network/payload/addr.go | 2 +- pkg/network/payload/addr_test.go | 2 +- pkg/network/payload/getblocks.go | 2 +- pkg/network/payload/getblocks_test.go | 2 +- pkg/network/payload/getheaders.go | 2 +- pkg/network/payload/getheaders_test.go | 2 +- pkg/network/payload/inventory.go | 2 +- pkg/network/payload/inventory_test.go | 2 +- pkg/network/peer.go | 2 +- pkg/network/server.go | 4 +- pkg/network/server_test.go | 2 +- pkg/network/tcp.go | 4 +- 21 files changed, 126 insertions(+), 37 deletions(-) create mode 100644 VERSION create mode 100644 circle.yml diff --git a/Makefile b/Makefile index d4a442545..26ffa0846 100644 --- a/Makefile +++ b/Makefile @@ -1,8 +1,23 @@ +BRANCH = "master" +VERSION = $(shell cat ./VERSION) + build: @go build -o ./bin/neo-go ./cli/main.go +check-version: + git fetch && (! git rev-list ${VERSION}) + deps: @glide install +push-tag: + git checkout ${BRANCH} + git pull origin ${BRANCH} + git tag ${VERSION} + git push origin ${BRANCH} --tags + test: - @go test $(glide nv) -cover \ No newline at end of file + @go test $(shell glide nv) -cover + +vet: + @go vet $(shell glide nv) \ No newline at end of file diff --git a/VERSION b/VERSION new file mode 100644 index 000000000..6c6aa7cb0 --- /dev/null +++ b/VERSION @@ -0,0 +1 @@ +0.1.0 \ No newline at end of file diff --git a/circle.yml b/circle.yml new file mode 100644 index 000000000..86ce09569 --- /dev/null +++ b/circle.yml @@ -0,0 +1,86 @@ +version: 2 +jobs: + install_deps: + working_directory: /go/src/github.com/CityOfZion/neo-go + docker: + - image: vidsyhq/go-builder:latest + steps: + - checkout + - restore_cache: + key: dependency-cache-{{ .Revision }} + - run: BUILD=false /scripts/build.sh + - save_cache: + key: dependency-cache-{{ .Revision }} + paths: + - vendor + test: + working_directory: /go/src/github.com/CityOfZion/neo-go + docker: + - image: vidsyhq/go-builder:latest + steps: + - checkout + - restore_cache: + key: dependency-cache-{{ .Revision }} + - run: make test + vet: + working_directory: /go/src/github.com/CityOfZion/neo-go + docker: + - image: vidsyhq/go-builder:latest + steps: + - checkout + - restore_cache: + key: dependency-cache-{{ .Revision }} + - run: make vet + check_version: + working_directory: /go/src/github.com/CityOfZion/neo-go + docker: + - image: vidsyhq/go-builder:latest + steps: + - checkout + - run: make check-version + build_cli: + working_directory: /go/src/github.com/CityOfZion/neo-go + docker: + - image: vidsyhq/go-builder:latest + steps: + - checkout + - run: make build + +workflows: + version: 2 + workflow: + jobs: + - install_deps: + filters: + tags: + only: /[0-9]+\.[0-9]+\.[0-9]+/ + branches: + ignore: master + - test: + requires: + - install_deps + filters: + tags: + only: /[0-9]+\.[0-9]+\.[0-9]+/ + branches: + ignore: master + - vet: + requires: + - install_deps + filters: + tags: + only: /[0-9]+\.[0-9]+\.[0-9]+/ + branches: + ignore: master + - check_version: + filters: + branches: + ignore: master + - build_cli: + requires: + - install_deps + filters: + tags: + only: /[0-9]+\.[0-9]+\.[0-9]+/ + branches: + ignore: master \ No newline at end of file diff --git a/cli/main.go b/cli/main.go index fff926922..c8b13fd1a 100644 --- a/cli/main.go +++ b/cli/main.go @@ -4,7 +4,7 @@ import ( "flag" "strings" - "github.com/anthdm/neo-go/pkg/network" + "github.com/CityOfZion/neo-go/pkg/network" ) var ( diff --git a/glide.lock b/glide.lock index a3815ad55..59f70a101 100644 --- a/glide.lock +++ b/glide.lock @@ -1,11 +1,4 @@ -hash: 054e4119c1d6deac9c76a3f6ecc319c80d2099d6dd4fd804b03ce47a9b1ccc86 -updated: 2018-02-01T17:18:00.958758Z -imports: -- name: github.com/anthdm/neo-go - version: 3b91a4808e67c54687a4ba90ac4caf584398cea4 - subpackages: - - pkg/core - - pkg/network - - pkg/network/payload - - pkg/util +hash: b1152abdd9a1fa1e70773cddcf54247d3fe3332602604f9f2233165ced02eeaf +updated: 2018-02-01T18:34:22.684905Z +imports: [] testImports: [] diff --git a/glide.yaml b/glide.yaml index dc88f8824..3b61baf8d 100644 --- a/glide.yaml +++ b/glide.yaml @@ -1,8 +1,2 @@ package: github.com/CityOfZion/neo-go -import: -- package: github.com/anthdm/neo-go - subpackages: - - pkg/core - - pkg/network - - pkg/network/payload - - pkg/util +import: [] diff --git a/pkg/core/block.go b/pkg/core/block.go index 5e8eece3c..9b82c6960 100644 --- a/pkg/core/block.go +++ b/pkg/core/block.go @@ -4,7 +4,7 @@ import ( "encoding/binary" "io" - . "github.com/anthdm/neo-go/pkg/util" + . "github.com/CityOfZion/neo-go/pkg/util" ) // Block represents one block in the chain. diff --git a/pkg/network/message.go b/pkg/network/message.go index 070cec320..3d9dc96a1 100644 --- a/pkg/network/message.go +++ b/pkg/network/message.go @@ -8,8 +8,8 @@ import ( "fmt" "io" - "github.com/anthdm/neo-go/pkg/core" - "github.com/anthdm/neo-go/pkg/network/payload" + "github.com/CityOfZion/neo-go/pkg/core" + "github.com/CityOfZion/neo-go/pkg/network/payload" ) const ( diff --git a/pkg/network/message_test.go b/pkg/network/message_test.go index 4b4e5608f..6b1c5b596 100644 --- a/pkg/network/message_test.go +++ b/pkg/network/message_test.go @@ -5,7 +5,7 @@ import ( "reflect" "testing" - "github.com/anthdm/neo-go/pkg/network/payload" + "github.com/CityOfZion/neo-go/pkg/network/payload" ) func TestMessageEncodeDecode(t *testing.T) { diff --git a/pkg/network/payload/addr.go b/pkg/network/payload/addr.go index b5376524a..26f2de4af 100644 --- a/pkg/network/payload/addr.go +++ b/pkg/network/payload/addr.go @@ -4,7 +4,7 @@ import ( "encoding/binary" "io" - "github.com/anthdm/neo-go/pkg/util" + "github.com/CityOfZion/neo-go/pkg/util" ) // AddrWithTime payload diff --git a/pkg/network/payload/addr_test.go b/pkg/network/payload/addr_test.go index c021b72df..8f2da310d 100644 --- a/pkg/network/payload/addr_test.go +++ b/pkg/network/payload/addr_test.go @@ -6,7 +6,7 @@ import ( "reflect" "testing" - "github.com/anthdm/neo-go/pkg/util" + "github.com/CityOfZion/neo-go/pkg/util" ) func TestEncodeDecodeAddr(t *testing.T) { diff --git a/pkg/network/payload/getblocks.go b/pkg/network/payload/getblocks.go index 81371cb25..f7a84a1c2 100644 --- a/pkg/network/payload/getblocks.go +++ b/pkg/network/payload/getblocks.go @@ -4,7 +4,7 @@ import ( "encoding/binary" "io" - . "github.com/anthdm/neo-go/pkg/util" + . "github.com/CityOfZion/neo-go/pkg/util" ) // HashStartStop contains fields and methods to be shared with the diff --git a/pkg/network/payload/getblocks_test.go b/pkg/network/payload/getblocks_test.go index c73772386..5c62edd96 100644 --- a/pkg/network/payload/getblocks_test.go +++ b/pkg/network/payload/getblocks_test.go @@ -6,7 +6,7 @@ import ( "reflect" "testing" - . "github.com/anthdm/neo-go/pkg/util" + . "github.com/CityOfZion/neo-go/pkg/util" ) func TestGetBlocksEncodeDecode(t *testing.T) { diff --git a/pkg/network/payload/getheaders.go b/pkg/network/payload/getheaders.go index 9ade274f7..ce71a626c 100644 --- a/pkg/network/payload/getheaders.go +++ b/pkg/network/payload/getheaders.go @@ -1,6 +1,6 @@ package payload -import "github.com/anthdm/neo-go/pkg/util" +import "github.com/CityOfZion/neo-go/pkg/util" // GetHeaders payload is the same as the "GetBlocks" payload. type GetHeaders struct { diff --git a/pkg/network/payload/getheaders_test.go b/pkg/network/payload/getheaders_test.go index 32cbb1b86..87a3e6672 100644 --- a/pkg/network/payload/getheaders_test.go +++ b/pkg/network/payload/getheaders_test.go @@ -6,7 +6,7 @@ import ( "reflect" "testing" - "github.com/anthdm/neo-go/pkg/util" + "github.com/CityOfZion/neo-go/pkg/util" ) func TestGetHeadersEncodeDecode(t *testing.T) { diff --git a/pkg/network/payload/inventory.go b/pkg/network/payload/inventory.go index 306995016..46310bfe5 100644 --- a/pkg/network/payload/inventory.go +++ b/pkg/network/payload/inventory.go @@ -4,7 +4,7 @@ import ( "encoding/binary" "io" - . "github.com/anthdm/neo-go/pkg/util" + . "github.com/CityOfZion/neo-go/pkg/util" ) // The node can broadcast the object information it owns by this message. diff --git a/pkg/network/payload/inventory_test.go b/pkg/network/payload/inventory_test.go index 774e33017..4cfba1ef5 100644 --- a/pkg/network/payload/inventory_test.go +++ b/pkg/network/payload/inventory_test.go @@ -6,7 +6,7 @@ import ( "reflect" "testing" - . "github.com/anthdm/neo-go/pkg/util" + . "github.com/CityOfZion/neo-go/pkg/util" ) func TestInventoryEncodeDecode(t *testing.T) { diff --git a/pkg/network/peer.go b/pkg/network/peer.go index 36a4f87f2..c83b7b24c 100644 --- a/pkg/network/peer.go +++ b/pkg/network/peer.go @@ -1,7 +1,7 @@ package network import ( - "github.com/anthdm/neo-go/pkg/util" + "github.com/CityOfZion/neo-go/pkg/util" ) // Peer is the local representation of a remote node. It's an interface that may diff --git a/pkg/network/server.go b/pkg/network/server.go index c18c11695..0d568482c 100644 --- a/pkg/network/server.go +++ b/pkg/network/server.go @@ -7,8 +7,8 @@ import ( "os" "time" - "github.com/anthdm/neo-go/pkg/network/payload" - "github.com/anthdm/neo-go/pkg/util" + "github.com/CityOfZion/neo-go/pkg/network/payload" + "github.com/CityOfZion/neo-go/pkg/util" ) const ( diff --git a/pkg/network/server_test.go b/pkg/network/server_test.go index f0ba721f7..115c2b74c 100644 --- a/pkg/network/server_test.go +++ b/pkg/network/server_test.go @@ -3,7 +3,7 @@ package network import ( "testing" - "github.com/anthdm/neo-go/pkg/network/payload" + "github.com/CityOfZion/neo-go/pkg/network/payload" ) func TestHandleVersion(t *testing.T) { diff --git a/pkg/network/tcp.go b/pkg/network/tcp.go index 648fe6274..4c9b13b4d 100644 --- a/pkg/network/tcp.go +++ b/pkg/network/tcp.go @@ -5,8 +5,8 @@ import ( "fmt" "net" - "github.com/anthdm/neo-go/pkg/network/payload" - "github.com/anthdm/neo-go/pkg/util" + "github.com/CityOfZion/neo-go/pkg/network/payload" + "github.com/CityOfZion/neo-go/pkg/util" ) func listenTCP(s *Server, port int) error { From 66c8fc801223faf1edc537bb26e66c5cd47ab1c2 Mon Sep 17 00:00:00 2001 From: Anthony De Meulemeester Date: Thu, 1 Feb 2018 21:28:45 +0100 Subject: [PATCH 10/10] merge original into coz repo (#10) merged with the original repo. --- VERSION | 2 +- pkg/core/block.go | 53 +++++++++++++++++++++-- pkg/core/block_test.go | 6 +++ pkg/core/blockchain.go | 26 ++++++++++++ pkg/core/blockchain_storer.go | 44 +++++++++++++++++++ pkg/network/server.go | 79 +++++++++++++++++++++++++++++++++-- pkg/network/tcp.go | 22 +++++----- 7 files changed, 211 insertions(+), 21 deletions(-) create mode 100644 pkg/core/block_test.go create mode 100644 pkg/core/blockchain.go create mode 100644 pkg/core/blockchain_storer.go diff --git a/VERSION b/VERSION index 6c6aa7cb0..341cf11fa 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.1.0 \ No newline at end of file +0.2.0 \ No newline at end of file diff --git a/pkg/core/block.go b/pkg/core/block.go index 9b82c6960..552528c2d 100644 --- a/pkg/core/block.go +++ b/pkg/core/block.go @@ -1,20 +1,24 @@ package core import ( + "bytes" + "crypto/sha256" "encoding/binary" "io" . "github.com/CityOfZion/neo-go/pkg/util" ) -// Block represents one block in the chain. -type Block struct { +// BlockBase holds the base info of a block +type BlockBase struct { Version uint32 // hash of the previous block. PrevBlock Uint256 // Root hash of a transaction list. MerkleRoot Uint256 - // timestamp + // The time stamp of each block must be later than previous block's time stamp. + // Generally the difference of two block's time stamp is about 15 seconds and imprecision is allowed. + // The height of the block must be exactly equal to the height of the previous block plus 1. Timestamp uint32 // height of the block Height uint32 @@ -22,14 +26,40 @@ type Block struct { Nonce uint64 // contract addresss of the next miner NextMiner Uint160 - // seperator ? fixed to 1 + // fixed to 1 _sep uint8 // Script used to validate the block Script *Witness +} + +// BlockHead holds the head info of a block +type BlockHead struct { + BlockBase + // fixed to 0 + _sep1 uint8 +} + +// Block represents one block in the chain. +type Block struct { + BlockBase // transaction list Transactions []*Transaction } +// encodeHashableFields will only encode the fields used for hashing. +// see Hash() for more information about the fields. +func (b *Block) encodeHashableFields(w io.Writer) error { + err := binary.Write(w, binary.LittleEndian, &b.Version) + err = binary.Write(w, binary.LittleEndian, &b.PrevBlock) + err = binary.Write(w, binary.LittleEndian, &b.MerkleRoot) + err = binary.Write(w, binary.LittleEndian, &b.Timestamp) + err = binary.Write(w, binary.LittleEndian, &b.Height) + err = binary.Write(w, binary.LittleEndian, &b.Nonce) + err = binary.Write(w, binary.LittleEndian, &b.NextMiner) + + return err +} + // EncodeBinary encodes the block to the given writer. func (b *Block) EncodeBinary(w io.Writer) error { return nil @@ -66,5 +96,20 @@ func (b *Block) DecodeBinary(r io.Reader) error { return err } +// Hash return the hash of the block. +// When calculating the hash value of the block, instead of calculating the entire block, +// only first seven fields in the block head will be calculated, which are +// version, PrevBlock, MerkleRoot, timestamp, and height, the nonce, NextMiner. +// Since MerkleRoot already contains the hash value of all transactions, +// the modification of transaction will influence the hash value of the block. +func (b *Block) Hash() (hash Uint256, err error) { + buf := new(bytes.Buffer) + if err = b.encodeHashableFields(buf); err != nil { + return + } + hash = sha256.Sum256(buf.Bytes()) + return +} + // Size implements the payload interface. func (b *Block) Size() uint32 { return 0 } diff --git a/pkg/core/block_test.go b/pkg/core/block_test.go new file mode 100644 index 000000000..88d5e11df --- /dev/null +++ b/pkg/core/block_test.go @@ -0,0 +1,6 @@ +package core + +import "testing" + +func TestBlockEncodeDecode(t *testing.T) { +} diff --git a/pkg/core/blockchain.go b/pkg/core/blockchain.go new file mode 100644 index 000000000..f8e84451e --- /dev/null +++ b/pkg/core/blockchain.go @@ -0,0 +1,26 @@ +package core + +// tuning parameters +const ( + secondsPerBlock = 15 +) + +var ( + genAmount = []int{8, 7, 6, 5, 4, 3, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1} +) + +// Blockchain holds the chain. +type Blockchain struct { + // Any object that satisfies the BlockchainStorer interface. + BlockchainStorer + + // index of the latest block. + currentHeight uint32 +} + +// NewBlockchain returns a pointer to a Blockchain. +func NewBlockchain(store BlockchainStorer) *Blockchain { + return &Blockchain{ + BlockchainStorer: store, + } +} diff --git a/pkg/core/blockchain_storer.go b/pkg/core/blockchain_storer.go new file mode 100644 index 000000000..6b874c0a0 --- /dev/null +++ b/pkg/core/blockchain_storer.go @@ -0,0 +1,44 @@ +package core + +import ( + "sync" + + "github.com/CityOfZion/neo-go/pkg/util" +) + +// BlockchainStorer is anything that can persist and retrieve the blockchain. +type BlockchainStorer interface { + HasBlock(util.Uint256) bool + GetBlockByHeight(uint32) (*Block, error) + GetBlockByHash(util.Uint256) (*Block, error) +} + +// MemoryStore is an in memory implementation of a BlockChainStorer. +type MemoryStore struct { + mtx *sync.RWMutex + blocks map[util.Uint256]*Block +} + +// NewMemoryStore returns a pointer to a MemoryStore object. +func NewMemoryStore() *MemoryStore { + return &MemoryStore{ + mtx: &sync.RWMutex{}, + blocks: map[util.Uint256]*Block{}, + } +} + +// HasBlock implements the BlockchainStorer interface. +func (s *MemoryStore) HasBlock(hash util.Uint256) bool { + _, ok := s.blocks[hash] + return ok +} + +// GetBlockByHash returns a block by its hash. +func (s *MemoryStore) GetBlockByHash(hash util.Uint256) (*Block, error) { + return nil, nil +} + +// GetBlockByHeight returns a block by its height. +func (s *MemoryStore) GetBlockByHeight(i uint32) (*Block, error) { + return nil, nil +} diff --git a/pkg/network/server.go b/pkg/network/server.go index 0d568482c..5bc145d36 100644 --- a/pkg/network/server.go +++ b/pkg/network/server.go @@ -5,8 +5,10 @@ import ( "log" "net" "os" + "sync" "time" + "github.com/CityOfZion/neo-go/pkg/core" "github.com/CityOfZion/neo-go/pkg/network/payload" "github.com/CityOfZion/neo-go/pkg/util" ) @@ -54,6 +56,46 @@ type Server struct { listener net.Listener // channel for safely responding the number of current connected peers. peerCountCh chan peerCount + // a list of hashes that + knownHashes protectedHashmap + // The blockchain. + bc *core.Blockchain +} + +type protectedHashmap struct { + *sync.RWMutex + hashes map[util.Uint256]bool +} + +func (m protectedHashmap) add(h util.Uint256) bool { + m.Lock() + defer m.Unlock() + + if _, ok := m.hashes[h]; !ok { + m.hashes[h] = true + return true + } + return false +} + +func (m protectedHashmap) remove(h util.Uint256) bool { + m.Lock() + defer m.Unlock() + + if _, ok := m.hashes[h]; ok { + delete(m.hashes, h) + return true + } + return false +} + +func (m protectedHashmap) has(h util.Uint256) bool { + m.RLock() + defer m.RUnlock() + + _, ok := m.hashes[h] + + return ok } // NewServer returns a pointer to a new server. @@ -76,6 +118,8 @@ func NewServer(net NetMode) *Server { net: net, quit: make(chan struct{}), peerCountCh: make(chan peerCount), + // knownHashes: protectedHashmap{}, + bc: core.NewBlockchain(core.NewMemoryStore()), } return s @@ -162,11 +206,11 @@ func (s *Server) handlePeerConnected(p Peer) { func (s *Server) handleVersionCmd(msg *Message, p Peer) *Message { version := msg.Payload.(*payload.Version) if s.id == version.Nonce { - p.disconnect() + // s.unregister <- p return nil } if p.addr().Port != version.Port { - p.disconnect() + // s.unregister <- p return nil } return newMessage(ModeDevNet, cmdVerack, nil) @@ -176,22 +220,46 @@ func (s *Server) handleGetaddrCmd(msg *Message, p Peer) *Message { return nil } +// The node can broadcast the object information it owns by this message. +// The message can be sent automatically or can be used to answer getbloks messages. func (s *Server) handleInvCmd(msg *Message, p Peer) *Message { inv := msg.Payload.(*payload.Inventory) if !inv.Type.Valid() { - p.disconnect() + s.unregister <- p return nil } if len(inv.Hashes) == 0 { - p.disconnect() + s.unregister <- p return nil } + // todo: only grab the hashes that we dont know. + payload := payload.NewInventory(inv.Type, inv.Hashes) resp := newMessage(s.net, cmdGetData, payload) return resp } +// handleBlockCmd processes the received block. +func (s *Server) handleBlockCmd(msg *Message, p Peer) { + block := msg.Payload.(*core.Block) + hash, err := block.Hash() + if err != nil { + // not quite sure what to do here. + // should we disconnect the client or just silently log and move on? + s.logger.Printf("failed to generate block hash: %s", err) + return + } + + fmt.Println(hash) + + if s.bc.HasBlock(hash) { + return + } +} + +// After receiving the getaddr message, the node returns an addr message as response +// and provides information about the known nodes on the network. func (s *Server) handleAddrCmd(msg *Message, p Peer) { addrList := msg.Payload.(*payload.AddressList) for _, addr := range addrList.Addrs { @@ -202,6 +270,9 @@ func (s *Server) handleAddrCmd(msg *Message, p Peer) { } } +func (s *Server) relayInventory(inv *payload.Inventory) { +} + // check if the addr is already connected to the server. func (s *Server) peerAlreadyConnected(addr net.Addr) bool { for peer := range s.peers { diff --git a/pkg/network/tcp.go b/pkg/network/tcp.go index 4c9b13b4d..dffb56b4d 100644 --- a/pkg/network/tcp.go +++ b/pkg/network/tcp.go @@ -3,6 +3,7 @@ package network import ( "bytes" "fmt" + "io" "net" "github.com/CityOfZion/neo-go/pkg/network/payload" @@ -64,6 +65,9 @@ func handleConnection(s *Server, conn net.Conn) { buf := make([]byte, 1024) for { _, err := conn.Read(buf) + if err == io.EOF { + break + } if err != nil { s.logger.Printf("conn read error: %s", err) break @@ -83,7 +87,7 @@ func handleConnection(s *Server, conn net.Conn) { func handleMessage(s *Server, p *TCPPeer) { // Disconnect the peer when we break out of the loop. defer func() { - p.disconnect() + s.unregister <- p }() for { @@ -97,14 +101,6 @@ func handleMessage(s *Server, p *TCPPeer) { resp := s.handleVersionCmd(msg, p) p.nonce = msg.Payload.(*payload.Version).Nonce p.send <- resp - - // after sending our version we want a "verack" and nothing else. - msg := <-p.receive - if msg.commandType() != cmdVerack { - break - } - // we can start the protocol now. - go s.sendLoop(p) case cmdAddr: s.handleAddrCmd(msg, p) case cmdGetAddr: @@ -113,11 +109,11 @@ func handleMessage(s *Server, p *TCPPeer) { resp := s.handleInvCmd(msg, p) p.send <- resp case cmdBlock: + s.handleBlockCmd(msg, p) case cmdConsensus: case cmdTX: case cmdVerack: - // disconnect the peer, verack should already be handled. - break + go s.sendLoop(p) case cmdGetHeaders: case cmdGetBlocks: case cmdGetData: @@ -174,9 +170,11 @@ func (p *TCPPeer) callGetaddr(msg *Message) { } // disconnect closes the send channel and the underlying connection. +// TODO: this needs some love. We will get send on closed channel. func (p *TCPPeer) disconnect() { - close(p.send) p.conn.Close() + close(p.send) + close(p.receive) } // writeLoop writes messages to the underlying TCP connection.