From de3137197a46d7d03fc2a1660c636dd9265d4fc2 Mon Sep 17 00:00:00 2001 From: BlockChainDev Date: Sun, 17 Mar 2019 23:02:34 +0000 Subject: [PATCH 01/24] remove extra base58 --- pkg/crypto/base58/base58.go | 48 ++++++++ pkg/wire/util/crypto/base58/base58.go | 126 --------------------- pkg/wire/util/crypto/base58/base58_test.go | 32 ------ pkg/wire/util/crypto/hash/hash.go | 83 -------------- pkg/wire/util/crypto/hash/hash_test.go | 62 ---------- 5 files changed, 48 insertions(+), 303 deletions(-) delete mode 100644 pkg/wire/util/crypto/base58/base58.go delete mode 100644 pkg/wire/util/crypto/base58/base58_test.go delete mode 100644 pkg/wire/util/crypto/hash/hash.go delete mode 100644 pkg/wire/util/crypto/hash/hash_test.go diff --git a/pkg/crypto/base58/base58.go b/pkg/crypto/base58/base58.go index bfac3d4f0..5d57dc8c4 100755 --- a/pkg/crypto/base58/base58.go +++ b/pkg/crypto/base58/base58.go @@ -1,8 +1,11 @@ package base58 import ( + "bytes" "fmt" "math/big" + + "github.com/CityOfZion/neo-go/pkg/crypto/hash" ) const prefix rune = '1' @@ -76,3 +79,48 @@ func Encode(bytes []byte) string { return encoded } + +// CheckDecode decodes the given string. +func CheckDecode(s string) (b []byte, err error) { + b, err = Decode(s) + if err != nil { + return nil, err + } + + for i := 0; i < len(s); i++ { + if s[i] != '1' { + break + } + b = append([]byte{0x00}, b...) + } + + if len(b) < 5 { + return nil, fmt.Errorf("Invalid base-58 check string: missing checksum") + } + + hash, err := hash.DoubleSha256(b[:len(b)-4]) + + if err != nil { + return nil, fmt.Errorf("Could not double sha256 data") + } + + if bytes.Compare(hash[0:4], b[len(b)-4:]) != 0 { + return nil, fmt.Errorf("Invalid base-58 check string: invalid checksum") + } + + // Strip the 4 byte long hash. + b = b[:len(b)-4] + + return b, nil +} + +// CheckEncode encodes b into a base-58 check encoded string. +func CheckEncode(b []byte) (string, error) { + hash, err := hash.DoubleSha256(b) + if err != nil { + return "", fmt.Errorf("Could not double sha256 data") + } + b = append(b, hash[0:4]...) + + return Encode(b), nil +} diff --git a/pkg/wire/util/crypto/base58/base58.go b/pkg/wire/util/crypto/base58/base58.go deleted file mode 100644 index 0e9710009..000000000 --- a/pkg/wire/util/crypto/base58/base58.go +++ /dev/null @@ -1,126 +0,0 @@ -package base58 - -import ( - "bytes" - "fmt" - "math/big" - - "github.com/CityOfZion/neo-go/pkg/wire/util/crypto/hash" -) - -const prefix rune = '1' - -var decodeMap = map[rune]int64{ - '1': 0, '2': 1, '3': 2, '4': 3, '5': 4, - '6': 5, '7': 6, '8': 7, '9': 8, 'A': 9, - 'B': 10, 'C': 11, 'D': 12, 'E': 13, 'F': 14, - 'G': 15, 'H': 16, 'J': 17, 'K': 18, 'L': 19, - 'M': 20, 'N': 21, 'P': 22, 'Q': 23, 'R': 24, - 'S': 25, 'T': 26, 'U': 27, 'V': 28, 'W': 29, - 'X': 30, 'Y': 31, 'Z': 32, 'a': 33, 'b': 34, - 'c': 35, 'd': 36, 'e': 37, 'f': 38, 'g': 39, - 'h': 40, 'i': 41, 'j': 42, 'k': 43, 'm': 44, - 'n': 45, 'o': 46, 'p': 47, 'q': 48, 'r': 49, - 's': 50, 't': 51, 'u': 52, 'v': 53, 'w': 54, - 'x': 55, 'y': 56, 'z': 57, -} - -// Decode decodes the base58 encoded string. -func Decode(s string) ([]byte, error) { - var ( - startIndex = 0 - zero = 0 - ) - for i, c := range s { - if c == prefix { - zero++ - } else { - startIndex = i - break - } - } - - var ( - n = big.NewInt(0) - div = big.NewInt(58) - ) - for _, c := range s[startIndex:] { - charIndex, ok := decodeMap[c] - if !ok { - return nil, fmt.Errorf( - "invalid character '%c' when decoding this base58 string: '%s'", c, s, - ) - } - n.Add(n.Mul(n, div), big.NewInt(charIndex)) - } - - out := n.Bytes() - buf := make([]byte, (zero + len(out))) - copy(buf[zero:], out[:]) - - return buf, nil -} - -// Encode encodes a byte slice to be a base58 encoded string. -func Encode(bytes []byte) string { - var ( - lookupTable = "123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz" - x = new(big.Int).SetBytes(bytes) - r = new(big.Int) - m = big.NewInt(58) - zero = big.NewInt(0) - encoded string - ) - - for x.Cmp(zero) > 0 { - x.QuoRem(x, m, r) - encoded = string(lookupTable[r.Int64()]) + encoded - } - - return encoded -} - -// CheckDecode decodes the given string. -func CheckDecode(s string) (b []byte, err error) { - b, err = Decode(s) - if err != nil { - return nil, err - } - - for i := 0; i < len(s); i++ { - if s[i] != '1' { - break - } - b = append([]byte{0x00}, b...) - } - - if len(b) < 5 { - return nil, fmt.Errorf("Invalid base-58 check string: missing checksum. -1") - } - - hash, err := hash.DoubleSha256(b[:len(b)-4]) - - if err != nil { - return nil, fmt.Errorf("Could not double sha256 data") - } - - if bytes.Compare(hash[0:4], b[len(b)-4:]) != 0 { - return nil, fmt.Errorf("Invalid base-58 check string: invalid checksum. -2") - } - - // Strip the 4 byte long hash. - b = b[:len(b)-4] - - return b, nil -} - -// CheckEncode encodes b into a base-58 check encoded string. -func CheckEncode(b []byte) (string, error) { - hash, err := hash.DoubleSha256(b) - if err != nil { - return "", fmt.Errorf("Could not double sha256 data") - } - b = append(b, hash[0:4]...) - - return Encode(b), nil -} diff --git a/pkg/wire/util/crypto/base58/base58_test.go b/pkg/wire/util/crypto/base58/base58_test.go deleted file mode 100644 index 524f0e55e..000000000 --- a/pkg/wire/util/crypto/base58/base58_test.go +++ /dev/null @@ -1,32 +0,0 @@ -package base58 - -import ( - "encoding/hex" - "testing" - - "github.com/stretchr/testify/assert" -) - -func TestDecode(t *testing.T) { - input := "1F1tAaz5x1HUXrCNLbtMDqcw6o5GNn4xqX" - - data, err := Decode(input) - if err != nil { - t.Fatal(err) - } - - expected := "0099bc78ba577a95a11f1a344d4d2ae55f2f857b989ea5e5e2" - actual := hex.EncodeToString(data) - assert.Equal(t, expected, actual) -} -func TestEncode(t *testing.T) { - input := "0099bc78ba577a95a11f1a344d4d2ae55f2f857b989ea5e5e2" - - inputBytes, _ := hex.DecodeString(input) - - data := Encode(inputBytes) - - expected := "F1tAaz5x1HUXrCNLbtMDqcw6o5GNn4xqX" // Removed the 1 as it is not checkEncoding - actual := data - assert.Equal(t, expected, actual) -} diff --git a/pkg/wire/util/crypto/hash/hash.go b/pkg/wire/util/crypto/hash/hash.go deleted file mode 100644 index 018885984..000000000 --- a/pkg/wire/util/crypto/hash/hash.go +++ /dev/null @@ -1,83 +0,0 @@ -package hash - -import ( - "crypto/sha256" - "io" - - "github.com/CityOfZion/neo-go/pkg/wire/util" - "golang.org/x/crypto/ripemd160" -) - -// Sha256 hashes the byte slice using sha256 -func Sha256(data []byte) (util.Uint256, error) { - var hash util.Uint256 - hasher := sha256.New() - hasher.Reset() - _, err := hasher.Write(data) - - hash, err = util.Uint256DecodeBytes(hasher.Sum(nil)) - if err != nil { - return hash, err - } - return hash, nil -} - -// DoubleSha256 hashes the underlying data twice using sha256 -func DoubleSha256(data []byte) (util.Uint256, error) { - var hash util.Uint256 - - h1, err := Sha256(data) - if err != nil { - return hash, err - } - - hash, err = Sha256(h1.Bytes()) - if err != nil { - return hash, err - } - return hash, nil -} - -// RipeMD160 hashes the underlying data using ripemd160 -func RipeMD160(data []byte) (util.Uint160, error) { - var hash util.Uint160 - hasher := ripemd160.New() - hasher.Reset() - _, err := io.WriteString(hasher, string(data)) - - hash, err = util.Uint160DecodeBytes(hasher.Sum(nil)) - if err != nil { - return hash, err - } - return hash, nil -} - -//Hash160 hashes the underlying data using sha256 then ripemd160 -func Hash160(data []byte) (util.Uint160, error) { - var hash util.Uint160 - h1, err := Sha256(data) - - h2, err := RipeMD160(h1.Bytes()) - - hash, err = util.Uint160DecodeBytes(h2.Bytes()) - - if err != nil { - return hash, err - } - return hash, nil -} - -// Checksum calculates the checksum of the byte slice using sha256 -func Checksum(data []byte) ([]byte, error) { - hash, err := Sum(data) - if err != nil { - return nil, err - } - return hash[:4], nil -} - -// Sum calculates the Sum of the data by using double sha256 -func Sum(b []byte) (util.Uint256, error) { - hash, err := DoubleSha256((b)) - return hash, err -} diff --git a/pkg/wire/util/crypto/hash/hash_test.go b/pkg/wire/util/crypto/hash/hash_test.go deleted file mode 100644 index aa23718f6..000000000 --- a/pkg/wire/util/crypto/hash/hash_test.go +++ /dev/null @@ -1,62 +0,0 @@ -package hash - -import ( - "encoding/hex" - "testing" - - "github.com/stretchr/testify/assert" -) - -func TestSha256(t *testing.T) { - input := []byte("hello") - data, err := Sha256(input) - - if err != nil { - t.Fatal(err) - } - expected := "2cf24dba5fb0a30e26e83b2ac5b9e29e1b161e5c1fa7425e73043362938b9824" - actual := hex.EncodeToString(data.Bytes()) // MARK: In the DecodeBytes function, there is a bytes reverse, not sure why? - - assert.Equal(t, expected, actual) -} - -func TestHashDoubleSha256(t *testing.T) { - input := []byte("hello") - data, err := DoubleSha256(input) - - if err != nil { - t.Fatal(err) - } - - firstSha, _ := Sha256(input) - doubleSha, _ := Sha256(firstSha.Bytes()) - expected := hex.EncodeToString(doubleSha.Bytes()) - - actual := hex.EncodeToString(data.Bytes()) - assert.Equal(t, expected, actual) -} - -func TestHashRipeMD160(t *testing.T) { - input := []byte("hello") - data, err := RipeMD160(input) - - if err != nil { - t.Fatal(err) - } - expected := "108f07b8382412612c048d07d13f814118445acd" - actual := hex.EncodeToString(data.Bytes()) - assert.Equal(t, expected, actual) -} - -func TestHash160(t *testing.T) { - input := "02cccafb41b220cab63fd77108d2d1ebcffa32be26da29a04dca4996afce5f75db" - publicKeyBytes, _ := hex.DecodeString(input) - data, err := Hash160(publicKeyBytes) - - if err != nil { - t.Fatal(err) - } - expected := "c8e2b685cc70ec96743b55beb9449782f8f775d8" - actual := hex.EncodeToString(data.Bytes()) - assert.Equal(t, expected, actual) -} From c1b6738bdb17602633ab098e89c42978fee932c9 Mon Sep 17 00:00:00 2001 From: decentralisedkev <37423678+decentralisedkev@users.noreply.github.com> Date: Mon, 18 Mar 2019 21:40:21 +0000 Subject: [PATCH 02/24] VM: Add basic vm (#166) * VM:Add abstract stack item * VM: Add stackItems; Array, Boolean, Int and ByteArray * VM: Add tests for stack item * VM: first pass at Random Access Stack object * VM: Add Sub, Mul, Mod LSH, RSH * VM: moved test helper functions into separate file * VM: removed helper functions from stack_test.go * Add conversions for bytearray and Int stack items * Add instructions file for vm * - Add guide to stack readme - Add testReadInt64 * Add Builder * Refactor Int, Boolean, ByteArray conversion * Add Context stack Item * Add Invocation stack - convenience RAS * rename testhelper to test_helper * Move opcode file * - Add `Add` OpCode - Add Opcode Function map * - Add test for math `Add` opcode - basic opcode execution * Add popTwoIntegers convenience func * Add `SUB` Opcode * Export Context Read methods - Return errors where failable * - Add `Op` to handleOP func signature - Add PushNBytes OPcode * remove error on NewBoolean - Expose underlying with Getter on Boolean StackItem - Add Equals method for ByteArray * Make Next() method on Context failable, refactor peekContext and Peek * Add ExecuteOp, Step and Run methods on the VM * Add Equal Opcode * Add THROWIFNOT Opcode * Add RET Opcode * Refactor PushNBytes Opcode * refactor Add, Sub to return VMSTATE add popTwoByteArrays helper function * Add basic tests for vm * clarify vm states * Add astack * [VM] Pass ResultStack to the opcode handlers * [VM] refactor handlers to have rstack as argument * [Stack] - Change RemoveCurrentContext for PopCurrentContext - Add CopTo method to stack * [VM] Add Result stack Len check in simple run test * [VM] fix typo * [Peer/Stall] Change seconds to milliseconds in test --- pkg/peer/stall/stall_test.go | 16 +-- pkg/vm/stack/Int.go | 94 +++++++++++++++++ pkg/vm/stack/Readme.md | 24 +++++ pkg/vm/stack/array.go | 13 +++ pkg/vm/stack/boolean.go | 26 +++++ pkg/vm/stack/builder.go | 177 ++++++++++++++++++++++++++++++++ pkg/vm/stack/bytearray.go | 71 +++++++++++++ pkg/vm/stack/context.go | 152 +++++++++++++++++++++++++++ pkg/vm/stack/instruction.go | 133 ++++++++++++++++++++++++ pkg/vm/stack/int_test.go | 72 +++++++++++++ pkg/vm/stack/invocationstack.go | 61 +++++++++++ pkg/vm/stack/stack.go | 150 +++++++++++++++++++++++++++ pkg/vm/stack/stack_test.go | 161 +++++++++++++++++++++++++++++ pkg/vm/stack/stackitem.go | 49 +++++++++ pkg/vm/stack/stackitem_test.go | 68 ++++++++++++ pkg/vm/stack/test_helper.go | 44 ++++++++ pkg/vm/state.go | 20 ++++ pkg/vm/vm.go | 72 +++++++++++++ pkg/vm/vm_ops.go | 21 ++++ pkg/vm/vm_ops_bitwise.go | 17 +++ pkg/vm/vm_ops_exceptions.go | 33 ++++++ pkg/vm/vm_ops_flow.go | 27 +++++ pkg/vm/vm_ops_maths.go | 70 +++++++++++++ pkg/vm/vm_ops_maths_test.go | 69 +++++++++++++ pkg/vm/vm_ops_stackmani.go | 19 ++++ pkg/vm/vm_test.go | 119 +++++++++++++++++++++ 26 files changed, 1770 insertions(+), 8 deletions(-) create mode 100644 pkg/vm/stack/Int.go create mode 100644 pkg/vm/stack/Readme.md create mode 100644 pkg/vm/stack/array.go create mode 100644 pkg/vm/stack/boolean.go create mode 100644 pkg/vm/stack/builder.go create mode 100644 pkg/vm/stack/bytearray.go create mode 100644 pkg/vm/stack/context.go create mode 100644 pkg/vm/stack/instruction.go create mode 100644 pkg/vm/stack/int_test.go create mode 100644 pkg/vm/stack/invocationstack.go create mode 100644 pkg/vm/stack/stack.go create mode 100644 pkg/vm/stack/stack_test.go create mode 100644 pkg/vm/stack/stackitem.go create mode 100644 pkg/vm/stack/stackitem_test.go create mode 100644 pkg/vm/stack/test_helper.go create mode 100644 pkg/vm/state.go create mode 100644 pkg/vm/vm.go create mode 100644 pkg/vm/vm_ops.go create mode 100644 pkg/vm/vm_ops_bitwise.go create mode 100644 pkg/vm/vm_ops_exceptions.go create mode 100644 pkg/vm/vm_ops_flow.go create mode 100644 pkg/vm/vm_ops_maths.go create mode 100644 pkg/vm/vm_ops_maths_test.go create mode 100644 pkg/vm/vm_ops_stackmani.go create mode 100644 pkg/vm/vm_test.go diff --git a/pkg/peer/stall/stall_test.go b/pkg/peer/stall/stall_test.go index b86412b2a..4d5494e12 100644 --- a/pkg/peer/stall/stall_test.go +++ b/pkg/peer/stall/stall_test.go @@ -12,8 +12,8 @@ import ( func TestAddRemoveMessage(t *testing.T) { - responseTime := 2 * time.Second - tickerInterval := 1 * time.Second + responseTime := 2 * time.Millisecond + tickerInterval := 1 * time.Millisecond d := NewDetector(responseTime, tickerInterval) d.AddMessage(command.GetAddr) @@ -51,15 +51,15 @@ loop: } func TestDeadlineWorks(t *testing.T) { - responseTime := 2 * time.Second - tickerInterval := 1 * time.Second + responseTime := 2 * time.Millisecond + tickerInterval := 1 * time.Millisecond d := NewDetector(responseTime, tickerInterval) mp := mockPeer{online: true, detector: d, lock: new(sync.RWMutex)} go mp.loop() d.AddMessage(command.GetAddr) - time.Sleep(responseTime + 1*time.Second) + time.Sleep(responseTime + 1*time.Millisecond) k := make(map[command.Type]time.Time) d.lock.RLock() @@ -70,12 +70,12 @@ func TestDeadlineWorks(t *testing.T) { mp.lock.RUnlock() } func TestDeadlineShouldNotBeEmpty(t *testing.T) { - responseTime := 10 * time.Second - tickerInterval := 1 * time.Second + responseTime := 10 * time.Millisecond + tickerInterval := 1 * time.Millisecond d := NewDetector(responseTime, tickerInterval) d.AddMessage(command.GetAddr) - time.Sleep(1 * time.Second) + time.Sleep(1 * time.Millisecond) k := make(map[command.Type]time.Time) d.lock.RLock() diff --git a/pkg/vm/stack/Int.go b/pkg/vm/stack/Int.go new file mode 100644 index 000000000..ac9a4cdba --- /dev/null +++ b/pkg/vm/stack/Int.go @@ -0,0 +1,94 @@ +package stack + +import "math/big" + +// Int represents an integer on the stack +type Int struct { + *abstractItem + val *big.Int +} + +// NewInt will convert a big integer into +// a StackInteger +func NewInt(val *big.Int) (*Int, error) { + return &Int{ + abstractItem: &abstractItem{}, + val: val, + }, nil +} + +// Equal will check if two integers hold equal value +func (i *Int) Equal(s *Int) bool { + if i.val.Cmp(s.val) != 0 { + return false + } + return true +} + +// Add will add two stackIntegers together +func (i *Int) Add(s *Int) (*Int, error) { + return &Int{ + val: new(big.Int).Add(i.val, s.val), + }, nil +} + +// Sub will subtract two stackIntegers together +func (i *Int) Sub(s *Int) (*Int, error) { + return &Int{ + val: new(big.Int).Sub(i.val, s.val), + }, nil +} + +// Mul will multiply two stackIntegers together +func (i *Int) Mul(s *Int) (*Int, error) { + return &Int{ + val: new(big.Int).Mul(i.val, s.val), + }, nil +} + +// Mod will take the mod of two stackIntegers together +func (i *Int) Mod(s *Int) (*Int, error) { + return &Int{ + val: new(big.Int).Mod(i.val, s.val), + }, nil +} + +// Rsh will shift the integer b to the right by `n` bits +func (i *Int) Rsh(n *Int) (*Int, error) { + return &Int{ + val: new(big.Int).Rsh(i.val, uint(n.val.Int64())), + }, nil +} + +// Lsh will shift the integer b to the left by `n` bits +func (i *Int) Lsh(n *Int) (*Int, error) { + return &Int{ + val: new(big.Int).Lsh(i.val, uint(n.val.Int64())), + }, nil +} + +// Integer will overwrite the default implementation +// to allow go to cast this item as an integer. +func (i *Int) Integer() (*Int, error) { + return i, nil +} + +// ByteArray override the default ByteArray method +// to convert a Integer into a byte Array +func (i *Int) ByteArray() (*ByteArray, error) { + b := i.val.Bytes() + dest := reverse(b) + return NewByteArray(dest), nil +} + +//Boolean override the default Boolean method +// to convert an Integer into a Boolean StackItem +func (i *Int) Boolean() (*Boolean, error) { + boolean := (i.val.Int64() != 0) + return NewBoolean(boolean), nil +} + +//Value returns the underlying big.Int +func (i *Int) Value() *big.Int { + return i.val +} diff --git a/pkg/vm/stack/Readme.md b/pkg/vm/stack/Readme.md new file mode 100644 index 000000000..2e1b6ba78 --- /dev/null +++ b/pkg/vm/stack/Readme.md @@ -0,0 +1,24 @@ +## VM - Stack + +- How do i implement a new StackItem? + +Answer: You add it's type to the Item interface, then you implement the default return method on the abstract stack item, this should be the behaviour of the stack item, if it is not the new type. Then you embed the abstract item in the new struct and override the method. + +For example, If I wanted to add a new type called `HashMap` + +type Item interface{ + HashMap()(*HashMap, error) +} + +func (a *abstractItem) HashMap() (*HashMap, error) { + return nil, errors.New(This stack item is not a hashmap) +} + +type HashMap struct { + *abstractItem + // Variables needed for hashmap +} + +func (h *HashMap) HashMap()(*HashMap, error) { + // logic to override default behaviour +} diff --git a/pkg/vm/stack/array.go b/pkg/vm/stack/array.go new file mode 100644 index 000000000..96fe876a4 --- /dev/null +++ b/pkg/vm/stack/array.go @@ -0,0 +1,13 @@ +package stack + +// Array represents an Array of stackItems on the stack +type Array struct { + *abstractItem + val []Item +} + +// Array overrides the default implementation +// by the abstractItem, returning an Array struct +func (a *Array) Array() (*Array, error) { + return a, nil +} diff --git a/pkg/vm/stack/boolean.go b/pkg/vm/stack/boolean.go new file mode 100644 index 000000000..66e3647e5 --- /dev/null +++ b/pkg/vm/stack/boolean.go @@ -0,0 +1,26 @@ +package stack + +// Boolean represents a boolean value on the stack +type Boolean struct { + *abstractItem + val bool +} + +//NewBoolean returns a new boolean stack item +func NewBoolean(val bool) *Boolean { + return &Boolean{ + &abstractItem{}, + val, + } +} + +// Boolean overrides the default implementation +// by the abstractItem, returning a Boolean struct +func (b *Boolean) Boolean() (*Boolean, error) { + return b, nil +} + +// Value returns the underlying boolean value +func (b *Boolean) Value() bool { + return b.val +} diff --git a/pkg/vm/stack/builder.go b/pkg/vm/stack/builder.go new file mode 100644 index 000000000..e50587aff --- /dev/null +++ b/pkg/vm/stack/builder.go @@ -0,0 +1,177 @@ +package stack + +import ( + "bytes" + "encoding/binary" + "errors" + "fmt" + "math/big" + + "github.com/CityOfZion/neo-go/pkg/wire/util" +) + +// Builder follows the builder pattern and will be used to build scripts +type Builder struct { + w *bytes.Buffer + err error +} + +// NewBuilder returns a new builder object +func NewBuilder() *Builder { + return &Builder{ + w: &bytes.Buffer{}, + err: nil, + } +} + +// Bytes returns the byte representation of the built buffer +func (br *Builder) Bytes() []byte { + return br.w.Bytes() +} + +// Emit a VM Opcode with data to the given buffer. +func (br *Builder) Emit(op Instruction, b []byte) *Builder { + if br.err != nil { + return br + } + br.err = br.w.WriteByte(byte(op)) + _, br.err = br.w.Write(b) + return br +} + +// EmitOpcode emits a single VM Opcode the given buffer. +func (br *Builder) EmitOpcode(op Instruction) *Builder { + if br.err != nil { + return br + } + br.err = br.w.WriteByte(byte(op)) + return br +} + +// EmitBool emits a bool type the given buffer. +func (br *Builder) EmitBool(ok bool) *Builder { + if br.err != nil { + return br + } + op := PUSHT + if !ok { + op = PUSHF + } + return br.EmitOpcode(op) +} + +// EmitInt emits a int type to the given buffer. +func (br *Builder) EmitInt(i int64) *Builder { + if br.err != nil { + return br + } + if i == -1 { + return br.EmitOpcode(PUSHM1) + } + if i == 0 { + return br.EmitOpcode(PUSHF) + } + if i > 0 && i < 16 { + val := Instruction(int(PUSH1) - 1 + int(i)) + return br.EmitOpcode(val) + } + + bInt := big.NewInt(i) + val := reverse(bInt.Bytes()) + return br.EmitBytes(val) +} + +// EmitString emits a string to the given buffer. +func (br *Builder) EmitString(s string) *Builder { + if br.err != nil { + return br + } + return br.EmitBytes([]byte(s)) +} + +// EmitBytes emits a byte array to the given buffer. +func (br *Builder) EmitBytes(b []byte) *Builder { + if br.err != nil { + return br + } + var ( + n = len(b) + ) + + if n <= int(PUSHBYTES75) { + return br.Emit(Instruction(n), b) + } else if n < 0x100 { + br.Emit(PUSHDATA1, []byte{byte(n)}) + } else if n < 0x10000 { + buf := make([]byte, 2) + binary.LittleEndian.PutUint16(buf, uint16(n)) + br.Emit(PUSHDATA2, buf) + } else { + buf := make([]byte, 4) + binary.LittleEndian.PutUint32(buf, uint32(n)) + br.Emit(PUSHDATA4, buf) + } + _, br.err = br.w.Write(b) + return br +} + +// EmitSyscall emits the syscall API to the given buffer. +// Syscall API string cannot be 0. +func (br *Builder) EmitSyscall(api string) *Builder { + if br.err != nil { + return br + } + if len(api) == 0 { + br.err = errors.New("syscall api cannot be of length 0") + } + buf := make([]byte, len(api)+1) + buf[0] = byte(len(api)) + copy(buf[1:], []byte(api)) + return br.Emit(SYSCALL, buf) +} + +// EmitCall emits a call Opcode with label to the given buffer. +func (br *Builder) EmitCall(op Instruction, label int16) *Builder { + return br.EmitJmp(op, label) +} + +// EmitJmp emits a jump Opcode along with label to the given buffer. +func (br *Builder) EmitJmp(op Instruction, label int16) *Builder { + if !isOpcodeJmp(op) { + br.err = fmt.Errorf("opcode %d is not a jump or call type", op) + } + buf := make([]byte, 2) + binary.LittleEndian.PutUint16(buf, uint16(label)) + return br.Emit(op, buf) +} + +// EmitAppCall emits an appcall, if tailCall is true, tailCall opcode will be +// emitted instead. +func (br *Builder) EmitAppCall(scriptHash util.Uint160, tailCall bool) *Builder { + op := APPCALL + if tailCall { + op = TAILCALL + } + return br.Emit(op, scriptHash.Bytes()) +} + +// EmitAppCallWithOperationAndData emits an appcall with the given operation and data. +func (br *Builder) EmitAppCallWithOperationAndData(w *bytes.Buffer, scriptHash util.Uint160, operation string, data []byte) *Builder { + br.EmitBytes(data) + br.EmitString(operation) + return br.EmitAppCall(scriptHash, false) +} + +// EmitAppCallWithOperation emits an appcall with the given operation. +func (br *Builder) EmitAppCallWithOperation(scriptHash util.Uint160, operation string) *Builder { + br.EmitBool(false) + br.EmitString(operation) + return br.EmitAppCall(scriptHash, false) +} + +func isOpcodeJmp(op Instruction) bool { + if op == JMP || op == JMPIFNOT || op == JMPIF || op == CALL { + return true + } + return false +} diff --git a/pkg/vm/stack/bytearray.go b/pkg/vm/stack/bytearray.go new file mode 100644 index 000000000..7d1c3c818 --- /dev/null +++ b/pkg/vm/stack/bytearray.go @@ -0,0 +1,71 @@ +package stack + +import ( + "bytes" + "errors" + "math/big" + "strconv" +) + +// ByteArray represents a slice of bytes on the stack +type ByteArray struct { + *abstractItem + val []byte +} + +//NewByteArray returns a ByteArray stack item +// given a byte slice +func NewByteArray(val []byte) *ByteArray { + return &ByteArray{ + &abstractItem{}, + val, + } +} + +//ByteArray overrides the default abstractItem Bytes array method +func (ba *ByteArray) ByteArray() (*ByteArray, error) { + return ba, nil +} + +//Equals returns true, if two bytearrays are equal +func (ba *ByteArray) Equals(other *ByteArray) *Boolean { + // If either are nil, return false + if ba == nil || other == nil { + return NewBoolean(false) + } + return NewBoolean(bytes.Equal(ba.val, other.val)) +} + +//Integer overrides the default Integer method to convert an +// ByteArray Into an integer +func (ba *ByteArray) Integer() (*Int, error) { + dest := reverse(ba.val) + integerVal := new(big.Int).SetBytes(dest) + return NewInt(integerVal) + +} + +// Boolean will convert a byte array into a boolean stack item +func (ba *ByteArray) Boolean() (*Boolean, error) { + boolean, err := strconv.ParseBool(string(ba.val)) + if err != nil { + return nil, errors.New("cannot convert byte array to a boolean") + } + return NewBoolean(boolean), nil +} + +// XXX: move this into a pkg/util/slice folder +// Go mod not working +func reverse(b []byte) []byte { + if len(b) < 2 { + return b + } + + dest := make([]byte, len(b)) + + for i, j := 0, len(b)-1; i < j+1; i, j = i+1, j-1 { + dest[i], dest[j] = b[j], b[i] + } + + return dest +} diff --git a/pkg/vm/stack/context.go b/pkg/vm/stack/context.go new file mode 100644 index 000000000..d381d74cb --- /dev/null +++ b/pkg/vm/stack/context.go @@ -0,0 +1,152 @@ +package stack + +import ( + "encoding/binary" + "errors" +) + +// Context represent the current execution context of the VM. +// context will be treated as stack item and placed onto the invocation stack +type Context struct { + *abstractItem + + // Instruction pointer. + ip int + + // The raw program script. + prog []byte + + // Breakpoints + breakPoints []int + + // Evaluation Stack + Estack RandomAccess + + // Alternative Stack + Astack RandomAccess +} + +// NewContext return a new Context object. +func NewContext(b []byte) *Context { + return &Context{ + abstractItem: &abstractItem{}, + ip: -1, + prog: b, + breakPoints: []int{}, + } +} + +// Context overrides the default implementation +// to return a context item +func (c *Context) Context() (*Context, error) { + return c, nil +} + +// Next return the next instruction to execute. +func (c *Context) Next() (Instruction, error) { + c.ip++ + if c.ip >= len(c.prog) { + return RET, errors.New("program pointer is more than the length of program. Returning RET OPCODE") + } + return Instruction(c.prog[c.ip]), nil +} + +// IP returns the absolute instruction without taking 0 into account. +// If that program starts the ip = 0 but IP() will return 1, cause its +// the first instruction. +func (c *Context) IP() int { + return c.ip + 1 +} + +// LenInstr returns the number of instructions loaded. +func (c *Context) LenInstr() int { + return len(c.prog) +} + +// CurrInstr returns the current instruction and opcode. +func (c *Context) CurrInstr() (int, Instruction) { + if c.ip < 0 { + return c.ip, Instruction(0x00) + } + return c.ip, Instruction(c.prog[c.ip]) +} + +// Copy returns an new exact copy of c. +func (c *Context) Copy() *Context { + return &Context{ + ip: c.ip, + prog: c.prog, + breakPoints: c.breakPoints, + } +} + +// Program returns the loaded program. +func (c *Context) Program() []byte { + return c.prog +} + +func (c *Context) atBreakPoint() bool { + for _, n := range c.breakPoints { + if n == c.ip { + return true + } + } + return false +} + +func (c *Context) String() string { + return "execution context" +} + +// ReadUint32 reads a uint32 from the script +func (c *Context) ReadUint32() uint32 { + start, end := c.IP(), c.IP()+4 + if end > len(c.prog) { + return 0 + } + val := binary.LittleEndian.Uint32(c.prog[start:end]) + c.ip += 4 + return val +} + +// ReadUint16 reads a uint16 from the script +func (c *Context) ReadUint16() uint16 { + start, end := c.IP(), c.IP()+2 + if end > len(c.prog) { + return 0 + } + val := binary.LittleEndian.Uint16(c.prog[start:end]) + c.ip += 2 + return val +} + +// ReadByte reads one byte from the script +func (c *Context) ReadByte() (byte, error) { + byt, err := c.ReadBytes(1) + if err != nil { + return 0, err + } + + return byt[0], nil +} + +// ReadBytes will read n bytes from the context +func (c *Context) ReadBytes(n int) ([]byte, error) { + start, end := c.IP(), c.IP()+n + if end > len(c.prog) { + return nil, errors.New("Too many bytes to read, pointer goes past end of program") + } + + out := make([]byte, n) + copy(out, c.prog[start:end]) + c.ip += n + return out, nil +} + +func (c *Context) readVarBytes() ([]byte, error) { + n, err := c.ReadByte() + if err != nil { + return nil, err + } + return c.ReadBytes(int(n)) +} diff --git a/pkg/vm/stack/instruction.go b/pkg/vm/stack/instruction.go new file mode 100644 index 000000000..1317a1e3e --- /dev/null +++ b/pkg/vm/stack/instruction.go @@ -0,0 +1,133 @@ +package stack + +// Instruction represents a operation code in the neovm +type Instruction byte + +// Viable list of supported instruction constants. +const ( + // Constants + PUSH0 Instruction = 0x00 + PUSHF Instruction = PUSH0 + PUSHBYTES1 Instruction = 0x01 + PUSHBYTES75 Instruction = 0x4B + PUSHDATA1 Instruction = 0x4C + PUSHDATA2 Instruction = 0x4D + PUSHDATA4 Instruction = 0x4E + PUSHM1 Instruction = 0x4F + PUSH1 Instruction = 0x51 + PUSHT Instruction = PUSH1 + PUSH2 Instruction = 0x52 + PUSH3 Instruction = 0x53 + PUSH4 Instruction = 0x54 + PUSH5 Instruction = 0x55 + PUSH6 Instruction = 0x56 + PUSH7 Instruction = 0x57 + PUSH8 Instruction = 0x58 + PUSH9 Instruction = 0x59 + PUSH10 Instruction = 0x5A + PUSH11 Instruction = 0x5B + PUSH12 Instruction = 0x5C + PUSH13 Instruction = 0x5D + PUSH14 Instruction = 0x5E + PUSH15 Instruction = 0x5F + PUSH16 Instruction = 0x60 + + // Flow control + NOP Instruction = 0x61 + JMP Instruction = 0x62 + JMPIF Instruction = 0x63 + JMPIFNOT Instruction = 0x64 + CALL Instruction = 0x65 + RET Instruction = 0x66 + APPCALL Instruction = 0x67 + SYSCALL Instruction = 0x68 + TAILCALL Instruction = 0x69 + + // Stack + DUPFROMALTSTACK Instruction = 0x6A + TOALTSTACK Instruction = 0x6B + FROMALTSTACK Instruction = 0x6C + XDROP Instruction = 0x6D + XSWAP Instruction = 0x72 + XTUCK Instruction = 0x73 + DEPTH Instruction = 0x74 + DROP Instruction = 0x75 + DUP Instruction = 0x76 + NIP Instruction = 0x77 + OVER Instruction = 0x78 + PICK Instruction = 0x79 + ROLL Instruction = 0x7A + ROT Instruction = 0x7B + SWAP Instruction = 0x7C + TUCK Instruction = 0x7D + + // Splice + CAT Instruction = 0x7E + SUBSTR Instruction = 0x7F + LEFT Instruction = 0x80 + RIGHT Instruction = 0x81 + SIZE Instruction = 0x82 + + // Bitwise logic + INVERT Instruction = 0x83 + AND Instruction = 0x84 + OR Instruction = 0x85 + XOR Instruction = 0x86 + EQUAL Instruction = 0x87 + + // Arithmetic + INC Instruction = 0x8B + DEC Instruction = 0x8C + SIGN Instruction = 0x8D + NEGATE Instruction = 0x8F + ABS Instruction = 0x90 + NOT Instruction = 0x91 + NZ Instruction = 0x92 + ADD Instruction = 0x93 + SUB Instruction = 0x94 + MUL Instruction = 0x95 + DIV Instruction = 0x96 + MOD Instruction = 0x97 + SHL Instruction = 0x98 + SHR Instruction = 0x99 + BOOLAND Instruction = 0x9A + BOOLOR Instruction = 0x9B + NUMEQUAL Instruction = 0x9C + NUMNOTEQUAL Instruction = 0x9E + LT Instruction = 0x9F + GT Instruction = 0xA0 + LTE Instruction = 0xA1 + GTE Instruction = 0xA2 + MIN Instruction = 0xA3 + MAX Instruction = 0xA4 + WITHIN Instruction = 0xA5 + + // Crypto + SHA1 Instruction = 0xA7 + SHA256 Instruction = 0xA8 + HASH160 Instruction = 0xA9 + HASH256 Instruction = 0xAA + CHECKSIG Instruction = 0xAC + CHECKMULTISIG Instruction = 0xAE + + // Array + ARRAYSIZE Instruction = 0xC0 + PACK Instruction = 0xC1 + UNPACK Instruction = 0xC2 + PICKITEM Instruction = 0xC3 + SETITEM Instruction = 0xC4 + NEWARRAY Instruction = 0xC5 + NEWSTRUCT Instruction = 0xC6 + APPEND Instruction = 0xC8 + REVERSE Instruction = 0xC9 + REMOVE Instruction = 0xCA + + // Exceptions + THROW Instruction = 0xF0 + THROWIFNOT Instruction = 0xF1 +) + +// Value returns the byte-value of the opcode. +func (i Instruction) Value() byte { + return byte(i) +} diff --git a/pkg/vm/stack/int_test.go b/pkg/vm/stack/int_test.go new file mode 100644 index 000000000..25d360183 --- /dev/null +++ b/pkg/vm/stack/int_test.go @@ -0,0 +1,72 @@ +package stack + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestAdd(t *testing.T) { + a := testMakeStackInt(t, 10) + b := testMakeStackInt(t, 20) + expected := testMakeStackInt(t, 30) + c, err := a.Add(b) + assert.Nil(t, err) + assert.Equal(t, true, expected.Equal(c)) +} +func TestSub(t *testing.T) { + a := testMakeStackInt(t, 30) + b := testMakeStackInt(t, 200) + expected := testMakeStackInt(t, 170) + c, err := b.Sub(a) + assert.Nil(t, err) + assert.Equal(t, true, expected.Equal(c)) +} +func TestMul(t *testing.T) { + a := testMakeStackInt(t, 10) + b := testMakeStackInt(t, 20) + expected := testMakeStackInt(t, 200) + c, err := a.Mul(b) + assert.Nil(t, err) + assert.Equal(t, true, expected.Equal(c)) +} +func TestMod(t *testing.T) { + a := testMakeStackInt(t, 10) + b := testMakeStackInt(t, 20) + expected := testMakeStackInt(t, 10) + c, err := a.Mod(b) + assert.Nil(t, err) + assert.Equal(t, true, expected.Equal(c)) +} +func TestLsh(t *testing.T) { + a := testMakeStackInt(t, 23) + b := testMakeStackInt(t, 8) + expected := testMakeStackInt(t, 5888) + c, err := a.Lsh(b) + assert.Nil(t, err) + assert.Equal(t, true, expected.Equal(c)) +} + +func TestRsh(t *testing.T) { + a := testMakeStackInt(t, 128) + b := testMakeStackInt(t, 3) + expected := testMakeStackInt(t, 16) + c, err := a.Rsh(b) + assert.Nil(t, err) + assert.Equal(t, true, expected.Equal(c)) +} + +func TestByteArrConversion(t *testing.T) { + + var num int64 = 100000 + + a := testMakeStackInt(t, num) + ba, err := a.ByteArray() + assert.Nil(t, err) + + have, err := ba.Integer() + assert.Nil(t, err) + + assert.Equal(t, num, have.val.Int64()) + +} diff --git a/pkg/vm/stack/invocationstack.go b/pkg/vm/stack/invocationstack.go new file mode 100644 index 000000000..49f058f62 --- /dev/null +++ b/pkg/vm/stack/invocationstack.go @@ -0,0 +1,61 @@ +package stack + +import "errors" + +// Invocation embeds a Random Access stack +// Providing helper methods for the context object +type Invocation struct{ RandomAccess } + +//NewInvocation will return a new +// Invocation stack +func NewInvocation() *Invocation { + return &Invocation{ + RandomAccess{ + vals: make([]Item, 0, StackAverageSize), + }, + } +} + +func (i *Invocation) peekContext(n uint16) (*Context, error) { + item, err := i.Peek(n) + if err != nil { + return nil, err + } + return item.Context() +} + +// CurrentContext returns the current context on the invocation stack +func (i *Invocation) CurrentContext() (*Context, error) { + return i.peekContext(0) +} + +// PopCurrentContext Pops a context item from the top of the stack +func (i *Invocation) PopCurrentContext() (*Context, error) { + item, err := i.Pop() + if err != nil { + return nil, err + } + ctx, err := item.Context() + if err != nil { + return nil, err + } + return ctx, err +} + +// CallingContext will return the cntext item +// that will be called next. +func (i *Invocation) CallingContext() (*Context, error) { + if i.Len() < 1 { + return nil, errors.New("Length of invocation stack is < 1, no calling context") + } + return i.peekContext(1) +} + +// EntryContext will return the context item that +// started the program +func (i *Invocation) EntryContext() (*Context, error) { + + // firstItemIndex refers to the first item that was popped on the stack + firstItemIndex := uint16(i.Len() - 1) // N.B. if this overflows because len is zero, then an error will be returned + return i.peekContext(firstItemIndex) +} diff --git a/pkg/vm/stack/stack.go b/pkg/vm/stack/stack.go new file mode 100644 index 000000000..8d1ac5b78 --- /dev/null +++ b/pkg/vm/stack/stack.go @@ -0,0 +1,150 @@ +package stack + +import ( + "errors" + "fmt" +) + +const ( + // StackAverageSize is used to set the capacity of the stack + // setting this number too low, will cause extra allocations + StackAverageSize = 20 +) + +// RandomAccess represents a Random Access Stack +type RandomAccess struct { + vals []Item +} + +// New will return a new random access stack +func New() *RandomAccess { + return &RandomAccess{ + vals: make([]Item, 0, StackAverageSize), + } +} + +// Items will return all items in the stack +func (ras *RandomAccess) items() []Item { + return ras.vals +} + +//Len will return the length of the stack +func (ras *RandomAccess) Len() int { + if ras.vals == nil { + return -1 + } + return len(ras.vals) +} + +// Clear will remove all items in the stack +func (ras *RandomAccess) Clear() { + ras.vals = make([]Item, 0, StackAverageSize) +} + +// Pop will remove the last stack item that was added +func (ras *RandomAccess) Pop() (Item, error) { + if len(ras.vals) == 0 { + return nil, errors.New("There are no items on the stack to pop") + } + if ras.vals == nil { + return nil, errors.New("Cannot pop from a nil stack") + } + + l := len(ras.vals) + item := ras.vals[l-1] + ras.vals = ras.vals[:l-1] + + return item, nil +} + +// Push will put a stack item onto the top of the stack +func (ras *RandomAccess) Push(item Item) *RandomAccess { + if ras.vals == nil { + ras.vals = make([]Item, 0, StackAverageSize) + } + + ras.vals = append(ras.vals, item) + + return ras +} + +// Insert will push a stackItem onto the stack at position `n` +// Note; index 0 is the top of the stack, which is the end of slice +func (ras *RandomAccess) Insert(n uint16, item Item) (*RandomAccess, error) { + + if n == 0 { + return ras.Push(item), nil + } + + if ras.vals == nil { + ras.vals = make([]Item, 0, StackAverageSize) + } + + // Check that we are not inserting out of the bounds + stackSize := uint16(len(ras.vals)) + if n > stackSize-1 { + return nil, fmt.Errorf("Tried to insert at index %d when length of stack is %d", n, len(ras.vals)) + } + + index := stackSize - n + + ras.vals = append(ras.vals, item) + copy(ras.vals[index:], ras.vals[index-1:]) + ras.vals[index] = item + + return ras, nil +} + +// Peek will check an element at a given index +// Note: 0 is the top of the stack, which is the end of the slice +func (ras *RandomAccess) Peek(n uint16) (Item, error) { + + stackSize := uint16(len(ras.vals)) + + if n == 0 { + index := stackSize - 1 + return ras.vals[index], nil + } + + if ras.Len() < 1 { + return nil, fmt.Errorf("cannot peak at a stack with no item, length of stack is %d", ras.Len()) + } + + // Check that we are not peeking out of the bounds + if n > stackSize-1 { + return nil, fmt.Errorf("Tried to peek at index %d when length of stack is %d", n, len(ras.vals)) + } + index := stackSize - n - 1 + + return ras.vals[index], nil +} + +// CopyTo will copy all of the stack items from `ras` into the stack that is passed as an argument +// XXX: once maxstacksize is implemented, we will return error if size goes over +// There will also be additional checks needed once stack isolation is added +func (ras *RandomAccess) CopyTo(stack *RandomAccess) error { + stack.vals = append(stack.vals, ras.vals...) + return nil +} + +// Convenience Functions + +// PopInt will remove the last stack item that was added +// And cast it to an integer +func (ras *RandomAccess) PopInt() (*Int, error) { + item, err := ras.Pop() + if err != nil { + return nil, err + } + return item.Integer() +} + +// PopByteArray will remove the last stack item that was added +// And cast it to an ByteArray +func (ras *RandomAccess) PopByteArray() (*ByteArray, error) { + item, err := ras.Pop() + if err != nil { + return nil, err + } + return item.ByteArray() +} diff --git a/pkg/vm/stack/stack_test.go b/pkg/vm/stack/stack_test.go new file mode 100644 index 000000000..246e983f4 --- /dev/null +++ b/pkg/vm/stack/stack_test.go @@ -0,0 +1,161 @@ +package stack + +import ( + "fmt" + "math/big" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestStackPushPop(t *testing.T) { + // Create two stack Integers + a, err := NewInt(big.NewInt(10)) + if err != nil { + t.Fail() + } + b, err := NewInt(big.NewInt(20)) + if err != nil { + t.Fail() + } + + // Create a new stack + testStack := New() + + // Push to stack + testStack.Push(a).Push(b) + + // There should only be two values on the stack + assert.Equal(t, 2, testStack.Len()) + + // Pop first element and it should be equal to b + stackElement, err := testStack.Pop() + if err != nil { + t.Fail() + } + item, err := stackElement.Integer() + if err != nil { + t.Fail() + } + assert.Equal(t, true, item.Equal(b)) + + // Pop second element and it should be equal to a + stackElement, err = testStack.Pop() + if err != nil { + t.Fail() + } + item, err = stackElement.Integer() + if err != nil { + t.Fail() + } + assert.Equal(t, true, item.Equal(a)) + + // We should get an error as there are nomore items left to pop + stackElement, err = testStack.Pop() + assert.NotNil(t, err) + +} + +// For this test to pass, we should get an error when popping from a nil stack +// and we should initialise and push an element if pushing to an empty stack +func TestPushPopNil(t *testing.T) { + + // stack is nil when initialised without New constructor + testStack := RandomAccess{} + + // Popping from nil stack + // - should give an error + // - element returned should be nil + stackElement, err := testStack.Pop() + assert.NotNil(t, err) + assert.Nil(t, stackElement) + + // stack should still be nil after failing to pop + assert.Nil(t, testStack.vals) + + // create a random test stack item + a, err := NewInt(big.NewInt(2)) + assert.Nil(t, err) + + // push random item to stack + testStack.Push(a) + + // push should initialise the stack and put one element on the stack + assert.Equal(t, 1, testStack.Len()) +} + +// Test passes if we can peek and modify an item +//without modifying the value on the stack +func TestStackPeekMutability(t *testing.T) { + + testStack := New() + + a, err := NewInt(big.NewInt(2)) + assert.Nil(t, err) + b, err := NewInt(big.NewInt(3)) + assert.Nil(t, err) + + testStack.Push(a).Push(b) + + peekedItem := testPeakInteger(t, testStack, 0) + assert.Equal(t, true, peekedItem.Equal(b)) + + // Check that by modifying the peeked value, + // we did not modify the item on the stack + peekedItem = a + peekedItem.val = big.NewInt(0) + + // Pop item from stack and check it is still the same + poppedItem := testPopInteger(t, testStack) + assert.Equal(t, true, poppedItem.Equal(b)) +} +func TestStackPeek(t *testing.T) { + + testStack := New() + + values := []int64{23, 45, 67, 89, 12, 344} + for _, val := range values { + a := testMakeStackInt(t, val) + testStack.Push(a) + } + + // i starts at 0, j starts at len(values)-1 + for i, j := 0, len(values)-1; j >= 0; i, j = i+1, j-1 { + + peekedItem := testPeakInteger(t, testStack, uint16(i)) + a := testMakeStackInt(t, values[j]) + + fmt.Printf("%#v\n", peekedItem.val.Int64()) + + assert.Equal(t, true, a.Equal(peekedItem)) + + } + +} + +func TestStackInsert(t *testing.T) { + + testStack := New() + + a := testMakeStackInt(t, 2) + b := testMakeStackInt(t, 4) + c := testMakeStackInt(t, 6) + + // insert on an empty stack should put element on top + _, err := testStack.Insert(0, a) + assert.Equal(t, err, nil) + _, err = testStack.Insert(0, b) + assert.Equal(t, err, nil) + _, err = testStack.Insert(1, c) + assert.Equal(t, err, nil) + + // Order should be [a,c,b] + pop1 := testPopInteger(t, testStack) + pop2 := testPopInteger(t, testStack) + pop3 := testPopInteger(t, testStack) + + assert.Equal(t, true, pop1.Equal(b)) + assert.Equal(t, true, pop2.Equal(c)) + assert.Equal(t, true, pop3.Equal(a)) + +} diff --git a/pkg/vm/stack/stackitem.go b/pkg/vm/stack/stackitem.go new file mode 100644 index 000000000..beed13363 --- /dev/null +++ b/pkg/vm/stack/stackitem.go @@ -0,0 +1,49 @@ +package stack + +import ( + "errors" +) + +//Item is an interface which represents object that can be placed on the stack +type Item interface { + Integer() (*Int, error) + Boolean() (*Boolean, error) + ByteArray() (*ByteArray, error) + Array() (*Array, error) + Context() (*Context, error) +} + +// Represents an `abstract` stack item +// which will hold default values for stack items +// this is intended to be embedded into types that you will use on the stack +type abstractItem struct{} + +// Integer is the default implementation for a stackItem +// Implements Item interface +func (a *abstractItem) Integer() (*Int, error) { + return nil, errors.New("This stack item is not an Integer") +} + +// Boolean is the default implementation for a stackItem +// Implements Item interface +func (a *abstractItem) Boolean() (*Boolean, error) { + return nil, errors.New("This stack item is not a Boolean") +} + +// ByteArray is the default implementation for a stackItem +// Implements Item interface +func (a *abstractItem) ByteArray() (*ByteArray, error) { + return nil, errors.New("This stack item is not a byte array") +} + +// Array is the default implementation for a stackItem +// Implements Item interface +func (a *abstractItem) Array() (*Array, error) { + return nil, errors.New("This stack item is not an array") +} + +// Context is the default implementation for a stackItem +// Implements Item interface +func (a *abstractItem) Context() (*Context, error) { + return nil, errors.New("This stack item is not of type context") +} diff --git a/pkg/vm/stack/stackitem_test.go b/pkg/vm/stack/stackitem_test.go new file mode 100644 index 000000000..741d6f530 --- /dev/null +++ b/pkg/vm/stack/stackitem_test.go @@ -0,0 +1,68 @@ +package stack + +import ( + "math/big" + "testing" + + "github.com/stretchr/testify/assert" +) + +// A simple test to ensure that by embedding the abstract interface +// we immediately become a stack item, with the default values set to nil +func TestInterfaceEmbedding(t *testing.T) { + + // Create an anonymous struct that embeds the abstractItem + a := struct { + *abstractItem + }{ + &abstractItem{}, + } + + // Since interface checking can be done at compile time. + // If he abstractItem did not implement all methods of our interface `Item` + // Then any struct which embeds it, will also not implement the Item interface. + // This test would then give errors, at compile time. + var Items []Item + Items = append(Items, a) + + // Default methods should give errors + // Here we just need to test against one of the methods in the interface + for _, element := range Items { + x, err := element.Integer() + assert.Nil(t, x) + assert.NotNil(t, err, nil) + } + +} + +// TestIntCasting is a simple test to test that the Integer method is overwritten +// from the abstractItem +func TestIntMethodOverride(t *testing.T) { + + testValues := []int64{0, 10, 200, 30, 90} + var Items []Item + + // Convert a range of int64s into Stack Integers + // Adding them into an array of StackItems + for _, num := range testValues { + stackInteger, err := NewInt(big.NewInt(num)) + if err != nil { + t.Fail() + } + Items = append(Items, stackInteger) + } + + // For each item, call the Integer method on the interface + // Which should return an integer and no error + // as the stack integer struct overrides that method + for i, element := range Items { + k, err := element.Integer() + if err != nil { + t.Fail() + } + if k.val.Cmp(big.NewInt(testValues[i])) != 0 { + t.Fail() + } + } + +} diff --git a/pkg/vm/stack/test_helper.go b/pkg/vm/stack/test_helper.go new file mode 100644 index 000000000..15c6f87de --- /dev/null +++ b/pkg/vm/stack/test_helper.go @@ -0,0 +1,44 @@ +package stack + +import ( + "bytes" + "encoding/binary" + "math/big" + "testing" + + "github.com/stretchr/testify/assert" +) + +// helper functions +func testPeakInteger(t *testing.T, tStack *RandomAccess, n uint16) *Int { + stackElement, err := tStack.Peek(n) + assert.Nil(t, err) + item, err := stackElement.Integer() + if err != nil { + t.Fail() + } + return item +} + +func testPopInteger(t *testing.T, tStack *RandomAccess) *Int { + stackElement, err := tStack.Pop() + assert.Nil(t, err) + item, err := stackElement.Integer() + if err != nil { + t.Fail() + } + return item +} + +func testMakeStackInt(t *testing.T, num int64) *Int { + a, err := NewInt(big.NewInt(num)) + assert.Nil(t, err) + return a +} + +func testReadInt64(data []byte) int64 { + var ret int64 + buf := bytes.NewBuffer(data) + binary.Read(buf, binary.LittleEndian, &ret) + return ret +} diff --git a/pkg/vm/state.go b/pkg/vm/state.go new file mode 100644 index 000000000..e6760c7c9 --- /dev/null +++ b/pkg/vm/state.go @@ -0,0 +1,20 @@ +package vm + +//Vmstate represents all possible states that the neo-vm can be in +type Vmstate byte + +// List of possible vm states +const ( + // NONE is the running state of the vm + // NONE signifies that the vm is ready to process an opcode + NONE = 0 + // HALT is a stopped state of the vm + // where the stop was signalled by the program completion + HALT = 1 << 0 + // FAULT is a stopped state of the vm + // where the stop was signalled by an error in the program + FAULT = 1 << 1 + // BREAK is a suspended state for the VM + // were the break was signalled by a breakpoint + BREAK = 1 << 2 +) diff --git a/pkg/vm/vm.go b/pkg/vm/vm.go new file mode 100644 index 000000000..e207a7ed6 --- /dev/null +++ b/pkg/vm/vm.go @@ -0,0 +1,72 @@ +package vm + +import ( + "fmt" + + "github.com/CityOfZion/neo-go/pkg/vm/stack" +) + +// VM represents an instance of a Neo Virtual Machine +type VM struct { + // ResultStack contains the results of + // the last evaluation stack before the program terminated + ResultStack stack.RandomAccess + // InvocationStack contains all of the contexts + // loaded into the vm + InvocationStack stack.Invocation + state Vmstate +} + +// NewVM will: +// Set the state of the VM to NONE +// instantiate a script as a new context +// Push the Context to the Invocation stack +func NewVM(script []byte) *VM { + ctx := stack.NewContext(script) + v := &VM{ + state: NONE, + } + v.InvocationStack.Push(ctx) + return v +} + +// Run loops over the current context by continuously stepping. +// Run breaks; once step returns an error or any state that is not NONE +func (v *VM) Run() (Vmstate, error) { + for { + state, err := v.step() + if err != nil || state != NONE { + return state, err + } + } +} + +// step will read `one` opcode from the script in the current context +// Then excute that opcode +func (v *VM) step() (Vmstate, error) { + // Get Current Context + ctx, err := v.InvocationStack.CurrentContext() + if err != nil { + return FAULT, err + } + // Read Opcode from context + op, _ := ctx.Next() // The only error that can occur from this, is if the pointer goes over the pointer + // In the NEO-VM specs, this is ignored and we return the RET opcode + // Execute OpCode + state, err := v.executeOp(stack.Instruction(op), ctx) + if err != nil { + return FAULT, err + } + return state, nil +} + +// ExecuteOp will execute one opcode on a given context. +// If the opcode is not registered, then an unknown opcode error will be returned +func (v *VM) executeOp(op stack.Instruction, ctx *stack.Context) (Vmstate, error) { + //Find function which handles that specific opcode + handleOp, ok := opFunc[op] + if !ok { + return FAULT, fmt.Errorf("unknown opcode entered %v", op) + } + return handleOp(op, ctx, &v.InvocationStack, &v.ResultStack) +} diff --git a/pkg/vm/vm_ops.go b/pkg/vm/vm_ops.go new file mode 100644 index 000000000..39b796389 --- /dev/null +++ b/pkg/vm/vm_ops.go @@ -0,0 +1,21 @@ +package vm + +import "github.com/CityOfZion/neo-go/pkg/vm/stack" + +type stackInfo func(op stack.Instruction, ctx *stack.Context, istack *stack.Invocation, rstack *stack.RandomAccess) (Vmstate, error) + +var opFunc = map[stack.Instruction]stackInfo{ + stack.ADD: Add, + stack.SUB: Sub, + stack.PUSHBYTES1: PushNBytes, + stack.PUSHBYTES75: PushNBytes, + stack.RET: RET, + stack.EQUAL: EQUAL, + stack.THROWIFNOT: THROWIFNOT, +} + +func init() { + for i := int(stack.PUSHBYTES1); i <= int(stack.PUSHBYTES75); i++ { + opFunc[stack.Instruction(i)] = PushNBytes + } +} diff --git a/pkg/vm/vm_ops_bitwise.go b/pkg/vm/vm_ops_bitwise.go new file mode 100644 index 000000000..350543fa2 --- /dev/null +++ b/pkg/vm/vm_ops_bitwise.go @@ -0,0 +1,17 @@ +package vm + +import "github.com/CityOfZion/neo-go/pkg/vm/stack" + +// Bitwise logic + +// EQUAL pushes true to the stack +// If the two top items on the stack are equal +func EQUAL(op stack.Instruction, ctx *stack.Context, istack *stack.Invocation, rstack *stack.RandomAccess) (Vmstate, error) { + + itemA, itemB, err := popTwoByteArrays(ctx) + if err != nil { + return FAULT, err + } + ctx.Estack.Push(itemA.Equals(itemB)) + return NONE, nil +} diff --git a/pkg/vm/vm_ops_exceptions.go b/pkg/vm/vm_ops_exceptions.go new file mode 100644 index 000000000..dd09cfb60 --- /dev/null +++ b/pkg/vm/vm_ops_exceptions.go @@ -0,0 +1,33 @@ +package vm + +import ( + "errors" + + "github.com/CityOfZion/neo-go/pkg/vm/stack" +) + +// vm exceptions + +// THROWIFNOT faults if the item on the top of the stack +// does not evaluate to true +// For specific logic on how a number of bytearray is evaluated can be seen +// from the boolean conversion methods on the stack items +func THROWIFNOT(op stack.Instruction, ctx *stack.Context, istack *stack.Invocation, rstack *stack.RandomAccess) (Vmstate, error) { + + // Pop item from top of stack + item, err := ctx.Estack.Pop() + if err != nil { + return FAULT, err + } + // Convert to a boolean + ok, err := item.Boolean() + if err != nil { + return FAULT, err + } + + // If false, throw + if !ok.Value() { + return FAULT, errors.New("item on top of stack evaluates to false") + } + return NONE, nil +} diff --git a/pkg/vm/vm_ops_flow.go b/pkg/vm/vm_ops_flow.go new file mode 100644 index 000000000..67ca4f825 --- /dev/null +++ b/pkg/vm/vm_ops_flow.go @@ -0,0 +1,27 @@ +package vm + +import ( + "github.com/CityOfZion/neo-go/pkg/vm/stack" +) + +// Flow control + +// RET Returns from the current context +// Returns HALT if there are nomore context's to run +func RET(op stack.Instruction, ctx *stack.Context, istack *stack.Invocation, rstack *stack.RandomAccess) (Vmstate, error) { + + // Pop current context from the Inovation stack + ctx, err := istack.PopCurrentContext() + if err != nil { + return FAULT, err + } + // If this was the last context, then we copy over the evaluation stack to the resultstack + // As the program is about to terminate, once we remove the context + if istack.Len() == 0 { + + err = ctx.Estack.CopyTo(rstack) + return HALT, err + } + + return NONE, nil +} diff --git a/pkg/vm/vm_ops_maths.go b/pkg/vm/vm_ops_maths.go new file mode 100644 index 000000000..a15596a30 --- /dev/null +++ b/pkg/vm/vm_ops_maths.go @@ -0,0 +1,70 @@ +package vm + +import ( + "github.com/CityOfZion/neo-go/pkg/vm/stack" +) + +// Add adds two stack Items together. +// Returns an error if either items cannot be casted to an integer +// or if integers cannot be added together +func Add(op stack.Instruction, ctx *stack.Context, istack *stack.Invocation, rstack *stack.RandomAccess) (Vmstate, error) { + + operandA, operandB, err := popTwoIntegers(ctx) + if err != nil { + return FAULT, err + } + res, err := operandA.Add(operandB) + if err != nil { + return FAULT, err + } + + ctx.Estack.Push(res) + + return NONE, nil +} + +// Sub subtracts two stack Items. +// Returns an error if either items cannot be casted to an integer +// or if integers cannot be subtracted together +func Sub(op stack.Instruction, ctx *stack.Context, istack *stack.Invocation, rstack *stack.RandomAccess) (Vmstate, error) { + + operandA, operandB, err := popTwoIntegers(ctx) + if err != nil { + return FAULT, err + } + res, err := operandB.Sub(operandA) + if err != nil { + return HALT, err + } + + ctx.Estack.Push(res) + + return NONE, nil +} + +func popTwoIntegers(ctx *stack.Context) (*stack.Int, *stack.Int, error) { + operandA, err := ctx.Estack.PopInt() + if err != nil { + return nil, nil, err + } + operandB, err := ctx.Estack.PopInt() + if err != nil { + return nil, nil, err + } + + return operandA, operandB, nil +} + +func popTwoByteArrays(ctx *stack.Context) (*stack.ByteArray, *stack.ByteArray, error) { + // Pop first stack item and cast as byte array + ba1, err := ctx.Estack.PopByteArray() + if err != nil { + return nil, nil, err + } + // Pop second stack item and cast as byte array + ba2, err := ctx.Estack.PopByteArray() + if err != nil { + return nil, nil, err + } + return ba1, ba2, nil +} diff --git a/pkg/vm/vm_ops_maths_test.go b/pkg/vm/vm_ops_maths_test.go new file mode 100644 index 000000000..4964e6923 --- /dev/null +++ b/pkg/vm/vm_ops_maths_test.go @@ -0,0 +1,69 @@ +package vm + +import ( + "math/big" + "testing" + + "github.com/CityOfZion/neo-go/pkg/vm/stack" + "github.com/stretchr/testify/assert" +) + +func TestAddOp(t *testing.T) { + + v := VM{} + + a, err := stack.NewInt(big.NewInt(20)) + if err != nil { + t.Fail() + } + b, err := stack.NewInt(big.NewInt(23)) + if err != nil { + t.Fail() + } + + ctx := stack.NewContext([]byte{}) + ctx.Estack.Push(a).Push(b) + + v.executeOp(stack.ADD, ctx) + + // Stack should have one item + assert.Equal(t, 1, ctx.Estack.Len()) + + item, err := ctx.Estack.PopInt() + if err != nil { + t.Fail() + } + + assert.Equal(t, int64(43), item.Value().Int64()) + +} + +func TestSubOp(t *testing.T) { + + v := VM{} + + a, err := stack.NewInt(big.NewInt(30)) + if err != nil { + t.Fail() + } + b, err := stack.NewInt(big.NewInt(40)) + if err != nil { + t.Fail() + } + + ctx := stack.NewContext([]byte{}) + ctx.Estack.Push(a).Push(b) + + v.executeOp(stack.SUB, ctx) + + // Stack should have one item + assert.Equal(t, 1, ctx.Estack.Len()) + + item, err := ctx.Estack.PopInt() + if err != nil { + t.Fail() + } + + assert.Equal(t, int64(-10), item.Value().Int64()) + +} diff --git a/pkg/vm/vm_ops_stackmani.go b/pkg/vm/vm_ops_stackmani.go new file mode 100644 index 000000000..f5e2ddc24 --- /dev/null +++ b/pkg/vm/vm_ops_stackmani.go @@ -0,0 +1,19 @@ +package vm + +import ( + "github.com/CityOfZion/neo-go/pkg/vm/stack" +) + +// Stack Manipulation Opcodes + +// PushNBytes will Read N Bytes from the script and push it onto the stack +func PushNBytes(op stack.Instruction, ctx *stack.Context, istack *stack.Invocation, rstack *stack.RandomAccess) (Vmstate, error) { + + val, err := ctx.ReadBytes(int(op)) + if err != nil { + return FAULT, err + } + ba := stack.NewByteArray(val) + ctx.Estack.Push(ba) + return NONE, nil +} diff --git a/pkg/vm/vm_test.go b/pkg/vm/vm_test.go new file mode 100644 index 000000000..adb8b5db3 --- /dev/null +++ b/pkg/vm/vm_test.go @@ -0,0 +1,119 @@ +package vm + +import ( + "fmt" + "testing" + + "github.com/CityOfZion/neo-go/pkg/vm/stack" + "github.com/stretchr/testify/assert" +) + +func TestPushAdd(t *testing.T) { + builder := stack.NewBuilder() + + // PUSH TWO NUMBER + // ADD THEM TOGETHER + builder.EmitInt(20).EmitInt(34).EmitOpcode(stack.ADD) + + // Pass program to VM + vm := NewVM(builder.Bytes()) + + // Execute first OPCODE + // Should be PUSH(20) + state, err := vm.step() + assert.Equal(t, NONE, int(state)) + assert.Nil(t, err) + + // We should have the number 20 on stack + ok := peekTopEStackIsValue(t, vm, 20) + assert.True(t, ok) + + // Excute second OPCODE + // Should be PUSH(34) + state, err = vm.step() + assert.Equal(t, NONE, int(state)) + assert.Nil(t, err) + + // We should have the number 34 at the top of the stack + ok = peekTopEStackIsValue(t, vm, 34) + assert.True(t, ok) + + // Excute third OPCODE + // Should Add both values on the stack + state, err = vm.step() + assert.Equal(t, NONE, int(state)) + assert.Nil(t, err) + + // We should now have one value on the stack + //It should be equal to 20+34 = 54 + ok = EstackLen(t, vm, 1) + assert.True(t, ok) + ok = peekTopEStackIsValue(t, vm, 54) + assert.True(t, ok) + + // If we try to step again, we should get a nil error and HALT + // because we have gone over the instruction pointer + // error is nil because when there are nomore instructions, the vm + // will add a RET opcode and return + state, err = vm.step() + assert.Equal(t, HALT, int(state)) + assert.Nil(t, err) + +} + +func TestSimpleRun(t *testing.T) { + + // Program pushes 20 and 34 to the stack + // Adds them together + // pushes 54 to the stack + // Checks if result of addition and 54 are equal + // Faults if not + + // Push(20) + // Push(34) + // Add + // Push(54) + // Equal + //THROWIFNOT + builder := stack.NewBuilder() + builder.EmitInt(20).EmitInt(34).EmitOpcode(stack.ADD) + builder.EmitInt(54).EmitOpcode(stack.EQUAL).EmitOpcode(stack.THROWIFNOT) + + // Pass program to VM + vm := NewVM(builder.Bytes()) + + // Runs vm with program + _, err := vm.Run() + assert.Nil(t, err) + + // ResultStack should be nil + assert.Equal(t, -1, vm.ResultStack.Len()) + +} + +// returns true if the value at the top of the evaluation stack is a integer +// and equals the value passed in +func peekTopEStackIsValue(t *testing.T, vm *VM, value int64) bool { + item := peakTopEstack(t, vm) + integer, err := item.Integer() + assert.Nil(t, err) + return value == integer.Value().Int64() +} + +// peaks the stack item on the top of the evaluation stack +// if the current context and returns it +func peakTopEstack(t *testing.T, vm *VM) stack.Item { + ctx, err := vm.InvocationStack.CurrentContext() + fmt.Println(err) + assert.Nil(t, err) + item, err := ctx.Estack.Peek(0) + assert.Nil(t, err) + return item +} + +// returns true if the total number of items on the evaluation stack is equal to value +func EstackLen(t *testing.T, vm *VM, value int) bool { + ctx, err := vm.InvocationStack.CurrentContext() + assert.Nil(t, err) + return value == ctx.Estack.Len() +} From 9a1ef386892474a0cafa476655e02ca342d43f48 Mon Sep 17 00:00:00 2001 From: decentralisedkev <37423678+decentralisedkev@users.noreply.github.com> Date: Thu, 21 Mar 2019 21:28:03 +0000 Subject: [PATCH 03/24] [Database] (#202) Remove unnecesary methods --- pkg/database/leveldb.go | 102 ++-------------------------------------- 1 file changed, 3 insertions(+), 99 deletions(-) diff --git a/pkg/database/leveldb.go b/pkg/database/leveldb.go index bb6c3ca0a..b90ffaf65 100644 --- a/pkg/database/leveldb.go +++ b/pkg/database/leveldb.go @@ -1,13 +1,6 @@ package database import ( - "bytes" - "encoding/binary" - "fmt" - - "github.com/CityOfZion/neo-go/pkg/wire/payload" - "github.com/CityOfZion/neo-go/pkg/wire/payload/transaction" - "github.com/CityOfZion/neo-go/pkg/wire/util" "github.com/syndtr/goleveldb/leveldb" "github.com/syndtr/goleveldb/leveldb/errors" ) @@ -32,29 +25,17 @@ type Database interface { Close() error } -var ( - // TX is the prefix used when inserting a tx into the db - TX = []byte("TX") - // HEADER is the prefix used when inserting a header into the db - HEADER = []byte("HEADER") - // LATESTHEADER is the prefix used when inserting the latests header into the db - LATESTHEADER = []byte("LH") - // UTXO is the prefix used when inserting a utxo into the db - UTXO = []byte("UTXO") -) - // New will return a new leveldb instance func New(path string) *LDB { db, err := leveldb.OpenFile(path, nil) + if err != nil { + return nil + } if _, corrupted := err.(*errors.ErrCorrupted); corrupted { db, err = leveldb.RecoverFile(path, nil) } - if err != nil { - return nil - } - return &LDB{ db, path, @@ -85,80 +66,3 @@ func (l *LDB) Delete(key []byte) error { func (l *LDB) Close() error { return l.db.Close() } - -// AddHeader adds a header into the database -func (l *LDB) AddHeader(header *payload.BlockBase) error { - - table := NewTable(l, HEADER) - - byt, err := header.Bytes() - if err != nil { - fmt.Println("Could not Get bytes from decoded BlockBase") - return nil - } - - fmt.Println("Adding Header, This should be batched!!!!") - - // This is the main mapping - //Key: HEADER+BLOCKHASH Value: contents of blockhash - key := header.Hash.Bytes() - err = table.Put(key, byt) - if err != nil { - fmt.Println("Error trying to add the original mapping into the DB for Header. Mapping is [Header]+[Hash]") - return err - } - - // This is the secondary mapping - // Key: HEADER + BLOCKHEIGHT Value: blockhash - - bh := uint32ToBytes(header.Index) - key = []byte(bh) - err = table.Put(key, header.Hash.Bytes()) - if err != nil { - return err - } - // This is the third mapping - // WARNING: This assumes that headers are adding in order. - return table.Put(LATESTHEADER, header.Hash.Bytes()) -} - -// AddTransactions adds a set of transactions into the database -func (l *LDB) AddTransactions(blockhash util.Uint256, txs []transaction.Transactioner) error { - - // SHOULD BE DONE IN BATCH!!!! - for i, tx := range txs { - buf := new(bytes.Buffer) - fmt.Println(tx.ID()) - tx.Encode(buf) - txByt := buf.Bytes() - txhash, err := tx.ID() - if err != nil { - fmt.Println("Error adding transaction with bytes", txByt) - return err - } - // This is the original mapping - // Key: [TX] + TXHASH - key := append(TX, txhash.Bytes()...) - l.Put(key, txByt) - - // This is the index - // Key: [TX] + BLOCKHASH + I <- i is the incrementer from the for loop - //Value : TXHASH - key = append(TX, blockhash.Bytes()...) - key = append(key, uint32ToBytes(uint32(i))...) - - err = l.Put(key, txhash.Bytes()) - if err != nil { - fmt.Println("Error could not add tx index into db") - return err - } - } - return nil -} - -// BigEndian -func uint32ToBytes(h uint32) []byte { - a := make([]byte, 4) - binary.BigEndian.PutUint32(a, h) - return a -} From e12255dd73dec62b5058093c72b14da6bec8a4c5 Mon Sep 17 00:00:00 2001 From: decentralisedkev <37423678+decentralisedkev@users.noreply.github.com> Date: Thu, 21 Mar 2019 23:18:02 +0000 Subject: [PATCH 04/24] [connmgr] Refactor Connmgr (#205) * [connmgr] - Refactor Connmgr - Remove un-needed async code - Add comment for Request --- pkg/connmgr/config.go | 25 ++++ pkg/connmgr/connmgr.go | 246 ++++++++++++++++++++++++++++++++++++ pkg/connmgr/connmgr_test.go | 107 ++++++++++++++++ pkg/connmgr/readme.md | 22 ++++ pkg/connmgr/request.go | 15 +++ 5 files changed, 415 insertions(+) create mode 100755 pkg/connmgr/config.go create mode 100755 pkg/connmgr/connmgr.go create mode 100755 pkg/connmgr/connmgr_test.go create mode 100755 pkg/connmgr/readme.md create mode 100755 pkg/connmgr/request.go diff --git a/pkg/connmgr/config.go b/pkg/connmgr/config.go new file mode 100755 index 000000000..66fada355 --- /dev/null +++ b/pkg/connmgr/config.go @@ -0,0 +1,25 @@ +package connmgr + +import ( + "net" +) + +// Config contains all methods which will be set by the caller to setup the connection manager. +type Config struct { + // GetAddress will return a single address for the connection manager to connect to + // This will be the source of addresses for the connection manager + GetAddress func() (string, error) + + // OnConnection is called by the connection manager when we successfully connect to a peer + // The caller should ideally inform the address manager that we have connected to this address in this function + OnConnection func(conn net.Conn, addr string) + + // OnAccept will take an established connection + OnAccept func(net.Conn) + + // AddressPort is the address port of the local node in the format "address:port" + AddressPort string + + // DialTimeout is the amount of time to wait, before we can disconnect a pending dialed connection + DialTimeout int +} diff --git a/pkg/connmgr/connmgr.go b/pkg/connmgr/connmgr.go new file mode 100755 index 000000000..a8e8442ac --- /dev/null +++ b/pkg/connmgr/connmgr.go @@ -0,0 +1,246 @@ +package connmgr + +import ( + "errors" + "fmt" + "net" + "net/http" + "time" +) + +var ( + // maxOutboundConn is the maximum number of active peers + // that the connection manager will try to have + maxOutboundConn = 10 + + // maxRetries is the maximum amount of successive retries that + // we can have before we stop dialing that peer + maxRetries = uint8(5) +) + +// Connmgr manages pending/active/failed cnnections +type Connmgr struct { + config Config + PendingList map[string]*Request + ConnectedList map[string]*Request + actionch chan func() +} + +//New creates a new connection manager +func New(cfg Config) *Connmgr { + cnnmgr := &Connmgr{ + cfg, + make(map[string]*Request), + make(map[string]*Request), + make(chan func(), 300), + } + + go func() { + + listener, err := net.Listen("tcp", cfg.AddressPort) + + if err != nil { + fmt.Println("Error connecting to outbound ", err) + } + + defer func() { + listener.Close() + }() + + for { + + conn, err := listener.Accept() + + if err != nil { + continue + } + go cfg.OnAccept(conn) + } + + }() + + return cnnmgr +} + +// NewRequest will make a new connection gets the address from address func in config +// Then dials it and assigns it to pending +func (c *Connmgr) NewRequest() error { + + // Fetch address + addr, err := c.config.GetAddress() + if err != nil { + return fmt.Errorf("error getting address " + err.Error()) + } + + r := &Request{ + Addr: addr, + } + return c.Connect(r) +} + +// Connect will dial the address in the Request +// Updating the request object depending on the outcome +func (c *Connmgr) Connect(r *Request) error { + + r.Retries++ + + conn, err := c.dial(r.Addr) + if err != nil { + c.failed(r) + return err + } + + r.Conn = conn + r.Inbound = true + + // r.Permanent is set by the address manager/caller. default is false + // The permanent connections will be the ones that are hardcoded, e.g seed3.ngd.network + // or are reliable. The connmgr will be more leniennt to permanent addresses as they have + // a track record or reputation of being reliable. + + return c.connected(r) +} + +//Disconnect will remove the request from the connected/pending list and close the connection +func (c *Connmgr) Disconnect(addr string) { + + var r *Request + + // fetch from connected list + r, ok := c.ConnectedList[addr] + if !ok { + // If not in connected, check pending + r, _ = c.PendingList[addr] + } + + c.disconnected(r) + +} + +// Dial is used to dial up connections given the addres and ip in the form address:port +func (c *Connmgr) dial(addr string) (net.Conn, error) { + dialTimeout := 1 * time.Second + conn, err := net.DialTimeout("tcp", addr, dialTimeout) + if err != nil { + if !isConnected() { + return nil, errors.New("Fatal Error: You do not seem to be connected to the internet") + } + return conn, err + } + return conn, nil +} +func (c *Connmgr) failed(r *Request) { + + c.actionch <- func() { + // priority to check if it is permanent or inbound + // if so then these peers are valuable in NEO and so we will just retry another time + if r.Inbound || r.Permanent { + multiplier := time.Duration(r.Retries * 10) + time.AfterFunc(multiplier*time.Second, + func() { + c.Connect(r) + }, + ) + // if not then we should check if this request has had maxRetries + // if it has then get a new address + // if not then call Connect on it again + } else if r.Retries > maxRetries { + if c.config.GetAddress != nil { + go c.NewRequest() + } + } else { + go c.Connect(r) + } + } + +} + +// Disconnected is called when a peer disconnects. +// we take the addr from peer, which is also it's key in the map +// and we use it to remove it from the connectedList +func (c *Connmgr) disconnected(r *Request) error { + + if r == nil { + // if object is nil, we return nil + return nil + } + + // if for some reason the underlying connection is not closed, close it + err := r.Conn.Close() + if err != nil { + return err + } + + // remove from any pending/connected list + delete(c.PendingList, r.Addr) + delete(c.ConnectedList, r.Addr) + + // If permanent,then lets retry + if r.Permanent { + return c.Connect(r) + } + + return nil +} + +//Connected is called when the connection manager makes a successful connection. +func (c *Connmgr) connected(r *Request) error { + + // This should not be the case, since we connected + if r == nil { + return errors.New("request object as nil inside of the connected function") + } + + // reset retries to 0 + r.Retries = 0 + + // add to connectedList + c.ConnectedList[r.Addr] = r + + // remove from pending if it was there + delete(c.PendingList, r.Addr) + + if c.config.OnConnection != nil { + c.config.OnConnection(r.Conn, r.Addr) + } + + return nil +} + +// Pending is synchronous, we do not want to continue with logic +// until we are certain it has been added to the pendingList +func (c *Connmgr) pending(r *Request) error { + + if r == nil { + return errors.New("request object is nil") + } + + c.PendingList[r.Addr] = r + + return nil +} + +// Run will start the connection manager +func (c *Connmgr) Run() error { + fmt.Println("Connection manager started") + go c.loop() + return nil +} + +func (c *Connmgr) loop() { + for { + select { + case f := <-c.actionch: + f() + } + } +} + +// https://stackoverflow.com/questions/50056144/check-for-internet-connection-from-application +func isConnected() (ok bool) { + _, err := http.Get("http://clients3.google.com/generate_204") + if err != nil { + return false + } + return true +} diff --git a/pkg/connmgr/connmgr_test.go b/pkg/connmgr/connmgr_test.go new file mode 100755 index 000000000..a060cf838 --- /dev/null +++ b/pkg/connmgr/connmgr_test.go @@ -0,0 +1,107 @@ +package connmgr + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestDial(t *testing.T) { + cfg := Config{ + GetAddress: nil, + OnConnection: nil, + OnAccept: nil, + AddressPort: "", + DialTimeout: 0, + } + + cm := New(cfg) + err := cm.Run() + assert.Equal(t, nil, err) + + ipport := "google.com:80" // google unlikely to go offline, a better approach to test Dialing is welcome. + + conn, err := cm.dial(ipport) + assert.Equal(t, nil, err) + assert.NotEqual(t, nil, conn) +} +func TestConnect(t *testing.T) { + cfg := Config{ + GetAddress: nil, + OnConnection: nil, + OnAccept: nil, + AddressPort: "", + DialTimeout: 0, + } + + cm := New(cfg) + cm.Run() + + ipport := "google.com:80" + + r := Request{Addr: ipport} + + err := cm.Connect(&r) + assert.Nil(t, err) + + assert.Equal(t, 1, len(cm.ConnectedList)) + +} +func TestNewRequest(t *testing.T) { + + address := "google.com:80" + + var getAddr = func() (string, error) { + return address, nil + } + + cfg := Config{ + GetAddress: getAddr, + OnConnection: nil, + OnAccept: nil, + AddressPort: "", + DialTimeout: 0, + } + + cm := New(cfg) + + cm.Run() + + cm.NewRequest() + + if _, ok := cm.ConnectedList[address]; ok { + assert.Equal(t, true, ok) + assert.Equal(t, 1, len(cm.ConnectedList)) + return + } + + assert.Fail(t, "Could not find the address in the connected lists") + +} +func TestDisconnect(t *testing.T) { + + address := "google.com:80" + + var getAddr = func() (string, error) { + return address, nil + } + + cfg := Config{ + GetAddress: getAddr, + OnConnection: nil, + OnAccept: nil, + AddressPort: "", + DialTimeout: 0, + } + + cm := New(cfg) + + cm.Run() + + cm.NewRequest() + + cm.Disconnect(address) + + assert.Equal(t, 0, len(cm.ConnectedList)) + +} diff --git a/pkg/connmgr/readme.md b/pkg/connmgr/readme.md new file mode 100755 index 000000000..d5d43fa01 --- /dev/null +++ b/pkg/connmgr/readme.md @@ -0,0 +1,22 @@ +# Package - Connection Manager + +## Responsibility + +- Manages the active, failed and pending connections for the node. + +## Features + +- Takes an Request, dials it and logs information based on the connectivity. + +- Retry failed connections. + +- Removable address source. The connection manager does not manage addresses, only connections. + + +## Usage + +The following methods are exposed from the Connection manager: + +- Connect(r *Request) : This takes a Request object and connects to it. It follow the same logic as NewRequest() however instead of getting the address from the datasource given upon initialisation, you directly feed the address you want to connect to. + +- Disconnect(addrport string) : Given an address:port, this will disconnect it, close the connection and remove it from the connected and pending list, if it was there. diff --git a/pkg/connmgr/request.go b/pkg/connmgr/request.go new file mode 100755 index 000000000..0d7def3c3 --- /dev/null +++ b/pkg/connmgr/request.go @@ -0,0 +1,15 @@ +package connmgr + +import ( + "net" +) + +// Request is a layer on top of connection and allows us to add metadata to the net.Conn +// that the connection manager can use to determine whether to retry and other useful heuristics +type Request struct { + Conn net.Conn + Addr string + Permanent bool + Inbound bool + Retries uint8 // should not be trying more than 255 tries +} From 6f496754fb035bf1a630feb151f4f804070c05ba Mon Sep 17 00:00:00 2001 From: dauTT <30392990+dauTT@users.noreply.github.com> Date: Sat, 23 Mar 2019 17:52:36 +0100 Subject: [PATCH 05/24] 1) Fixed String method in Uint256 (#208) 2) Added ReverseString Method to Uint256 --- pkg/wire/payload/block_test.go | 8 ++++---- pkg/wire/payload/mheaders_test.go | 4 ++-- pkg/wire/payload/transaction/claim_test.go | 4 ++-- pkg/wire/payload/transaction/contract_test.go | 6 +++--- pkg/wire/payload/transaction/enrollment_test.go | 2 +- pkg/wire/payload/transaction/invocation_test.go | 2 +- pkg/wire/payload/transaction/miner_test.go | 2 +- pkg/wire/payload/transaction/publish_test.go | 2 +- pkg/wire/payload/transaction/register_test.go | 4 ++-- pkg/wire/payload/transaction/state_test.go | 4 ++-- pkg/wire/util/uint256.go | 5 +++++ pkg/wire/util/uint256_test.go | 4 ++-- 12 files changed, 26 insertions(+), 21 deletions(-) diff --git a/pkg/wire/payload/block_test.go b/pkg/wire/payload/block_test.go index 2de7cdb4e..88c479c81 100644 --- a/pkg/wire/payload/block_test.go +++ b/pkg/wire/payload/block_test.go @@ -46,13 +46,13 @@ func TestBlockDecodeEncode(t *testing.T) { for _, tx := range b.Txs { switch t := tx.(type) { case *transaction.Contract: - hashes = append(hashes, t.Hash.String()) + hashes = append(hashes, t.Hash.ReverseString()) case *transaction.Miner: - hashes = append(hashes, t.Hash.String()) + hashes = append(hashes, t.Hash.ReverseString()) case *transaction.Claim: - hashes = append(hashes, t.Hash.String()) + hashes = append(hashes, t.Hash.ReverseString()) case *transaction.Invocation: - hashes = append(hashes, t.Hash.String()) + hashes = append(hashes, t.Hash.ReverseString()) } } diff --git a/pkg/wire/payload/mheaders_test.go b/pkg/wire/payload/mheaders_test.go index a03f43486..7527b2908 100644 --- a/pkg/wire/payload/mheaders_test.go +++ b/pkg/wire/payload/mheaders_test.go @@ -50,7 +50,7 @@ func TestAddAndEncodeHeaders(t *testing.T) { err := msgHeaders.Headers[0].createHash() assert.Equal(t, nil, err) // Hash being correct, automatically verifies that the fields are encoded properly - assert.Equal(t, "f3c4ec44c07eccbda974f1ee34bc6654ab6d3f22cd89c2e5c593a16d6cc7e6e8", msgHeaders.Headers[0].Hash.String()) + assert.Equal(t, "f3c4ec44c07eccbda974f1ee34bc6654ab6d3f22cd89c2e5c593a16d6cc7e6e8", msgHeaders.Headers[0].Hash.ReverseString()) } @@ -70,7 +70,7 @@ func TestEncodeDecode(t *testing.T) { header := headerMsg.Headers[0] err = header.createHash() - assert.Equal(t, "f3c4ec44c07eccbda974f1ee34bc6654ab6d3f22cd89c2e5c593a16d6cc7e6e8", header.Hash.String()) + assert.Equal(t, "f3c4ec44c07eccbda974f1ee34bc6654ab6d3f22cd89c2e5c593a16d6cc7e6e8", header.Hash.ReverseString()) buf := new(bytes.Buffer) diff --git a/pkg/wire/payload/transaction/claim_test.go b/pkg/wire/payload/transaction/claim_test.go index fb049fb58..53ae8ba79 100644 --- a/pkg/wire/payload/transaction/claim_test.go +++ b/pkg/wire/payload/transaction/claim_test.go @@ -26,9 +26,9 @@ func TestEncodeDecodeClaim(t *testing.T) { assert.Equal(t, 1, int(len(c.Claims))) claim := c.Claims[0] - assert.Equal(t, "497037a4c5e0a9ea1721e06f9d5e9aec183d11f2824ece93285729370f3a1baf", claim.PrevHash.String()) + assert.Equal(t, "497037a4c5e0a9ea1721e06f9d5e9aec183d11f2824ece93285729370f3a1baf", claim.PrevHash.ReverseString()) assert.Equal(t, uint16(0), claim.PrevIndex) - assert.Equal(t, "abf142faf539c340e42722b5b34b505cf4fd73185fed775784e37c2c5ef1b866", c.Hash.String()) + assert.Equal(t, "abf142faf539c340e42722b5b34b505cf4fd73185fed775784e37c2c5ef1b866", c.Hash.ReverseString()) // Encode buf := new(bytes.Buffer) diff --git a/pkg/wire/payload/transaction/contract_test.go b/pkg/wire/payload/transaction/contract_test.go index ea0393301..17c4de77f 100644 --- a/pkg/wire/payload/transaction/contract_test.go +++ b/pkg/wire/payload/transaction/contract_test.go @@ -27,12 +27,12 @@ func TestEncodeDecodeContract(t *testing.T) { input := c.Inputs[0] - assert.Equal(t, "eec17cc828d6ede932b57e4eaf79c2591151096a7825435cd67f498f9fa98d88", input.PrevHash.String()) + assert.Equal(t, "eec17cc828d6ede932b57e4eaf79c2591151096a7825435cd67f498f9fa98d88", input.PrevHash.ReverseString()) assert.Equal(t, 0, int(input.PrevIndex)) assert.Equal(t, int64(70600000000), c.Outputs[0].Amount) - assert.Equal(t, "c56f33fc6ecfcd0c225c4ab356fee59390af8560be0e930faebe74a6daff7c9b", c.Outputs[0].AssetID.String()) + assert.Equal(t, "c56f33fc6ecfcd0c225c4ab356fee59390af8560be0e930faebe74a6daff7c9b", c.Outputs[0].AssetID.ReverseString()) assert.Equal(t, "a8666b4830229d6a1a9b80f6088059191c122d2b", c.Outputs[0].ScriptHash.String()) - assert.Equal(t, "bdf6cc3b9af12a7565bda80933a75ee8cef1bc771d0d58effc08e4c8b436da79", c.Hash.String()) + assert.Equal(t, "bdf6cc3b9af12a7565bda80933a75ee8cef1bc771d0d58effc08e4c8b436da79", c.Hash.ReverseString()) // Encode buf := new(bytes.Buffer) diff --git a/pkg/wire/payload/transaction/enrollment_test.go b/pkg/wire/payload/transaction/enrollment_test.go index cf76af8e7..b2f0a3a16 100644 --- a/pkg/wire/payload/transaction/enrollment_test.go +++ b/pkg/wire/payload/transaction/enrollment_test.go @@ -26,5 +26,5 @@ func TestEncodeDecodeEnrollment(t *testing.T) { assert.Equal(t, nil, err) assert.Equal(t, rawtx, hex.EncodeToString(buf.Bytes())) - assert.Equal(t, "988832f693785dcbcb8d5a0e9d5d22002adcbfb1eb6bbeebf8c494fff580e147", enroll.Hash.String()) + assert.Equal(t, "988832f693785dcbcb8d5a0e9d5d22002adcbfb1eb6bbeebf8c494fff580e147", enroll.Hash.ReverseString()) } diff --git a/pkg/wire/payload/transaction/invocation_test.go b/pkg/wire/payload/transaction/invocation_test.go index 82ce2fcb6..fcd6a9818 100644 --- a/pkg/wire/payload/transaction/invocation_test.go +++ b/pkg/wire/payload/transaction/invocation_test.go @@ -34,7 +34,7 @@ func TestEncodeDecodeInvoc(t *testing.T) { assert.Equal(t, "31363a30373a3032203a2030333366616431392d643638322d343035382d626437662d313563393331323434336538", hex.EncodeToString(attr2.Data)) assert.Equal(t, "050034e23004141ad842821c7341d5a32b17d7177a1750d30014ca14628c9e5bc6a9346ca6bcdf050ceabdeb2bdc774953c1087472616e736665726703e1df72015bdef1a1b9567d4700635f23b1f406f1", hex.EncodeToString(i.Script)) - assert.Equal(t, "b2a22cd9dd7636ae23e25576866cd1d9e2f3d85a85e80874441f085cd60006d1", i.Hash.String()) + assert.Equal(t, "b2a22cd9dd7636ae23e25576866cd1d9e2f3d85a85e80874441f085cd60006d1", i.Hash.ReverseString()) // Encode buf := new(bytes.Buffer) diff --git a/pkg/wire/payload/transaction/miner_test.go b/pkg/wire/payload/transaction/miner_test.go index d053ce414..fa22ebcd4 100644 --- a/pkg/wire/payload/transaction/miner_test.go +++ b/pkg/wire/payload/transaction/miner_test.go @@ -24,7 +24,7 @@ func TestEncodeDecodeMiner(t *testing.T) { assert.Equal(t, types.Miner, m.Type) assert.Equal(t, uint32(571397116), m.Nonce) - assert.Equal(t, "a1f219dc6be4c35eca172e65e02d4591045220221b1543f1a4b67b9e9442c264", m.Hash.String()) + assert.Equal(t, "a1f219dc6be4c35eca172e65e02d4591045220221b1543f1a4b67b9e9442c264", m.Hash.ReverseString()) // Encode buf := new(bytes.Buffer) diff --git a/pkg/wire/payload/transaction/publish_test.go b/pkg/wire/payload/transaction/publish_test.go index 1c9bb1de1..e27a44777 100644 --- a/pkg/wire/payload/transaction/publish_test.go +++ b/pkg/wire/payload/transaction/publish_test.go @@ -28,6 +28,6 @@ func TestEncodeDecodePublish(t *testing.T) { assert.Equal(t, nil, err) assert.Equal(t, rawtx, hex.EncodeToString(buf.Bytes())) - assert.Equal(t, "5467a1fc8723ceffa8e5ee59399b02eea1df6fbaa53768c6704b90b960d223fa", publ.Hash.String()) + assert.Equal(t, "5467a1fc8723ceffa8e5ee59399b02eea1df6fbaa53768c6704b90b960d223fa", publ.Hash.ReverseString()) } diff --git a/pkg/wire/payload/transaction/register_test.go b/pkg/wire/payload/transaction/register_test.go index d5492a6d8..da8bfdc45 100644 --- a/pkg/wire/payload/transaction/register_test.go +++ b/pkg/wire/payload/transaction/register_test.go @@ -28,7 +28,7 @@ func TestEncodeDecodeRegister(t *testing.T) { assert.Equal(t, nil, err) assert.Equal(t, rawtx, hex.EncodeToString(buf.Bytes())) - assert.Equal(t, "0c092117b4ba47b81001712425e6e7f760a637695eaf23741ba335925b195ecd", reg.Hash.String()) + assert.Equal(t, "0c092117b4ba47b81001712425e6e7f760a637695eaf23741ba335925b195ecd", reg.Hash.ReverseString()) } func TestEncodeDecodeGenesisRegister(t *testing.T) { @@ -50,5 +50,5 @@ func TestEncodeDecodeGenesisRegister(t *testing.T) { assert.Equal(t, nil, err) assert.Equal(t, rawtx, hex.EncodeToString(buf.Bytes())) - assert.Equal(t, "c56f33fc6ecfcd0c225c4ab356fee59390af8560be0e930faebe74a6daff7c9b", reg.Hash.String()) + assert.Equal(t, "c56f33fc6ecfcd0c225c4ab356fee59390af8560be0e930faebe74a6daff7c9b", reg.Hash.ReverseString()) } diff --git a/pkg/wire/payload/transaction/state_test.go b/pkg/wire/payload/transaction/state_test.go index 209d9b9c0..d585daf76 100644 --- a/pkg/wire/payload/transaction/state_test.go +++ b/pkg/wire/payload/transaction/state_test.go @@ -25,7 +25,7 @@ func TestEncodeDecodeState(t *testing.T) { assert.Equal(t, 1, len(s.Inputs)) input := s.Inputs[0] - assert.Equal(t, "a192cbabc6d613ecfcce43fd09e9197556ca5cf7d4bd1f6c65726ea9f08441cb", input.PrevHash.String()) + assert.Equal(t, "a192cbabc6d613ecfcce43fd09e9197556ca5cf7d4bd1f6c65726ea9f08441cb", input.PrevHash.ReverseString()) assert.Equal(t, uint16(0), input.PrevIndex) assert.Equal(t, 1, len(s.Descriptors)) @@ -43,5 +43,5 @@ func TestEncodeDecodeState(t *testing.T) { assert.Equal(t, nil, err) assert.Equal(t, rawtxBytes, buf.Bytes()) - assert.Equal(t, "8abf5ebdb9a8223b12109513647f45bd3c0a6cf1a6346d56684cff71ba308724", s.Hash.String()) + assert.Equal(t, "8abf5ebdb9a8223b12109513647f45bd3c0a6cf1a6346d56684cff71ba308724", s.Hash.ReverseString()) } diff --git a/pkg/wire/util/uint256.go b/pkg/wire/util/uint256.go index e7a5e73fc..fc3fe86ca 100644 --- a/pkg/wire/util/uint256.go +++ b/pkg/wire/util/uint256.go @@ -63,6 +63,11 @@ func (u Uint256) Equals(other Uint256) bool { // String implements the stringer interface. func (u Uint256) String() string { + return hex.EncodeToString(u.Bytes()) +} + +// ReverseString displays a reverse string representation of Uint256. +func (u Uint256) ReverseString() string { return hex.EncodeToString(slice.Reverse(u.Bytes())) } diff --git a/pkg/wire/util/uint256_test.go b/pkg/wire/util/uint256_test.go index 120bee380..ae6b8a438 100644 --- a/pkg/wire/util/uint256_test.go +++ b/pkg/wire/util/uint256_test.go @@ -13,7 +13,7 @@ func TestUint256DecodeString(t *testing.T) { if err != nil { t.Fatal(err) } - assert.Equal(t, hexStr, val.Reverse().String()) + assert.Equal(t, hexStr, val.String()) } func TestUint256DecodeBytes(t *testing.T) { @@ -26,7 +26,7 @@ func TestUint256DecodeBytes(t *testing.T) { if err != nil { t.Fatal(err) } - assert.Equal(t, hexStr, val.Reverse().String()) + assert.Equal(t, hexStr, val.String()) } func TestUInt256Equals(t *testing.T) { From 30e5aa8f480474a087afca9edad3e1eb73491672 Mon Sep 17 00:00:00 2001 From: decentralisedkev <37423678+decentralisedkev@users.noreply.github.com> Date: Sat, 23 Mar 2019 16:57:05 +0000 Subject: [PATCH 06/24] Add Chain saving functionality to Database (#206) * [database] - Add Prefix method to interface - Convert leveldb error to `database error` - Be explicit with prefixedKey in `Table` as slices can be pointers * [protocol] - Add stringer method to protocol * [Chaindb] - Added saveBlock() which will allow us to save a block into the database. The block is broken up into transactions and Headers. The headers are saved as is. The transactions are saved as is, then the utxos in the transactions are collected to make the utxo db. - Verification for blocks and transactions will reside in the same package. Note that the save methods are all unexported, while the Get methods are exported. Making it so that any can call a get method, but only code in this package may save to the database. The other code which will reside in this package will be code verification logic. * [chaindb] - Added saveHeader function which saveHeaders uses - Update the latest header, each time we save a header instead of after a batch. This is so that we can call saveHeader without saveHeaders. This functionality can be rolled back if the performance of updating the header after a batch is significant - small refactor in test code --- pkg/chain/chaindb.go | 372 ++++++++++++++++++++++++++++++++++ pkg/chain/chaindb_test.go | 201 ++++++++++++++++++ pkg/database/leveldb.go | 62 +++++- pkg/database/leveldb_test.go | 33 ++- pkg/database/table.go | 22 +- pkg/peer/peer.go | 6 +- pkg/wire/protocol/protocol.go | 12 ++ 7 files changed, 679 insertions(+), 29 deletions(-) create mode 100644 pkg/chain/chaindb.go create mode 100644 pkg/chain/chaindb_test.go diff --git a/pkg/chain/chaindb.go b/pkg/chain/chaindb.go new file mode 100644 index 000000000..9b16b366a --- /dev/null +++ b/pkg/chain/chaindb.go @@ -0,0 +1,372 @@ +package chain + +import ( + "bufio" + "bytes" + "encoding/binary" + + "github.com/CityOfZion/neo-go/pkg/database" + "github.com/CityOfZion/neo-go/pkg/wire/payload" + "github.com/CityOfZion/neo-go/pkg/wire/payload/transaction" + "github.com/CityOfZion/neo-go/pkg/wire/util" +) + +var ( + // TX is the prefix used when inserting a tx into the db + TX = []byte("TX") + // HEADER is the prefix used when inserting a header into the db + HEADER = []byte("HE") + // LATESTHEADER is the prefix used when inserting the latests header into the db + LATESTHEADER = []byte("LH") + // UTXO is the prefix used when inserting a utxo into the db + UTXO = []byte("UT") + // LATESTBLOCK is the prefix used when inserting the latest block into the db + LATESTBLOCK = []byte("LB") + // BLOCKHASHTX is the prefix used when linking a blockhash to a given tx + BLOCKHASHTX = []byte("BT") + // BLOCKHASHHEIGHT is the prefix used when linking a blockhash to it's height + // This is linked both ways + BLOCKHASHHEIGHT = []byte("BH") + // SCRIPTHASHUTXO is the prefix used when linking a utxo to a scripthash + // This is linked both ways + SCRIPTHASHUTXO = []byte("SU") +) + +// Chaindb is a wrapper around the db interface which adds an extra block chain specific layer on top. +type Chaindb struct { + db database.Database +} + +// This should not be exported for other callers. +// It is safe-guarded by the chain's verification logic +func (c *Chaindb) saveBlock(blk payload.Block, genesis bool) error { + + latestBlockTable := database.NewTable(c.db, LATESTBLOCK) + hashHeightTable := database.NewTable(c.db, BLOCKHASHHEIGHT) + + // Save Txs and link to block hash + err := c.saveTXs(blk.Txs, blk.Hash.Bytes(), genesis) + if err != nil { + return err + } + + // LINK block height to hash - Both ways + // This allows us to fetch a block using it's hash or it's height + // Given the height, we will search the table to get the hash + // We can then fetch all transactions in the tx table, which match that block hash + height := uint32ToBytes(blk.Index) + err = hashHeightTable.Put(height, blk.Hash.Bytes()) + if err != nil { + return err + } + + err = hashHeightTable.Put(blk.Hash.Bytes(), height) + if err != nil { + return err + } + + // Add block as latest block + // This also acts a Commit() for the block. + // If an error occured, then this will be set to the previous block + // This is useful because if the node suddently shut down while saving and the database was not corrupted + // Then the node will see the latestBlock as the last saved block, and re-download the faulty block + // Note: We check for the latest block on startup + return latestBlockTable.Put([]byte(""), blk.Hash.Bytes()) +} + +// Saves a tx and links each tx to the block it was found in +// This should never be exported. Only way to add a tx, is through it's block +func (c *Chaindb) saveTXs(txs []transaction.Transactioner, blockHash []byte, genesis bool) error { + + for txIndex, tx := range txs { + err := c.saveTx(tx, uint32(txIndex), blockHash, genesis) + if err != nil { + return err + } + } + return nil +} + +func (c *Chaindb) saveTx(tx transaction.Transactioner, txIndex uint32, blockHash []byte, genesis bool) error { + + txTable := database.NewTable(c.db, TX) + blockTxTable := database.NewTable(c.db, BLOCKHASHTX) + + // Save the whole tx using it's hash a key + // In order to find a tx in this table, we need to know it's hash + txHash, err := tx.ID() + if err != nil { + return err + } + err = txTable.Put(txHash.Bytes(), tx.Bytes()) + if err != nil { + return err + } + + // LINK TXhash to block + // This allows us to fetch a tx by just knowing what block it was in + // This is useful for when we want to re-construct a block from it's hash + // In order to ge the tx, we must do a prefix search on blockHash + // This will return a set of txHashes. + //We can then use these hashes to search the txtable for the tx's we need + key := bytesConcat(blockHash, uint32ToBytes(txIndex)) + err = blockTxTable.Put(key, txHash.Bytes()) + if err != nil { + return err + } + + // Save all of the utxos in a transaction + // We do this additional save so that we can form a utxo database + // and know when a transaction is a double spend. + utxos := tx.UTXOs() + for utxoIndex, utxo := range utxos { + err := c.saveUTXO(utxo, uint16(utxoIndex), txHash.Bytes(), blockHash) + if err != nil { + return err + } + } + + // Do not check for spent utxos on the genesis block + if genesis { + return nil + } + + // Remove all spent utxos + // We do this so that once an output has been spent + // It will be removed from the utxo database and cannot be spent again + // If the output was never in the utxo database, this function will return an error + txos := tx.TXOs() + for _, txo := range txos { + err := c.removeUTXO(txo) + if err != nil { + return err + } + } + return nil +} + +// saveUTxo will save a utxo and link it to it's transaction and block +func (c *Chaindb) saveUTXO(utxo *transaction.Output, utxoIndex uint16, txHash, blockHash []byte) error { + + utxoTable := database.NewTable(c.db, UTXO) + scripthashUTXOTable := database.NewTable(c.db, SCRIPTHASHUTXO) + + // This is quite messy, we should (if possible) find a way to pass a Writer and Reader interface + // Encode utxo into a buffer + buf := new(bytes.Buffer) + bw := &util.BinWriter{W: buf} + if utxo.Encode(bw); bw.Err != nil { + return bw.Err + } + + // Save UTXO + // In order to find a utxo in the utxoTable + // One must know the txHash that the utxo was in + key := bytesConcat(txHash, uint16ToBytes(utxoIndex)) + if err := utxoTable.Put(key, buf.Bytes()); err != nil { + return err + } + + // LINK utxo to scripthash + // This allows us to find a utxo with the scriptHash + // Since the key starts with scriptHash, we can look for the scriptHash prefix + // and find all utxos for a given scriptHash. + // Additionally, we can search for all utxos for a certain user in a certain block with scriptHash+blockHash + // But this may not be of use to us. However, note that we cannot have just the scriptHash with the utxoIndex + // as this may not be unique. If Kim/Dautt agree, we can change blockHash to blockHeight, which allows us + // To get all utxos above a certain blockHeight. Question is; Would this be useful? + newKey := bytesConcat(utxo.ScriptHash.Bytes(), blockHash, uint16ToBytes(utxoIndex)) + if err := scripthashUTXOTable.Put(newKey, key); err != nil { + return err + } + if err := scripthashUTXOTable.Put(key, newKey); err != nil { + return err + } + return nil +} + +// Remove +func (c *Chaindb) removeUTXO(txo *transaction.Input) error { + + utxoTable := database.NewTable(c.db, UTXO) + scripthashUTXOTable := database.NewTable(c.db, SCRIPTHASHUTXO) + + // Remove spent utxos from utxo database + key := bytesConcat(txo.PrevHash.Bytes(), uint16ToBytes(txo.PrevIndex)) + err := utxoTable.Delete(key) + if err != nil { + return err + } + + // Remove utxos from scripthash table + otherKey, err := scripthashUTXOTable.Get(key) + if err != nil { + return err + } + if err := scripthashUTXOTable.Delete(otherKey); err != nil { + return err + } + if err := scripthashUTXOTable.Delete(key); err != nil { + return err + } + + return nil +} + +// saveHeaders will save a set of headers into the database +func (c *Chaindb) saveHeaders(headers []*payload.BlockBase) error { + + for _, hdr := range headers { + err := c.saveHeader(hdr) + if err != nil { + return err + } + } + return nil +} + +// saveHeader saves a header into the database and updates the latest header +// The headers are saved with their `blockheights` as Key +// If we want to search for a header, we need to know it's index +// Alternatively, we can search the hashHeightTable with the block index to get the hash +// If the block has been saved. +// The reason why headers are saved with their index as Key, is so that we can +// increment the key to find out what block we should fetch next during the initial +// block download, when we are saving thousands of headers +func (c *Chaindb) saveHeader(hdr *payload.BlockBase) error { + + headerTable := database.NewTable(c.db, HEADER) + latestHeaderTable := database.NewTable(c.db, LATESTHEADER) + + index := uint32ToBytes(hdr.Index) + + byt, err := hdr.Bytes() + if err != nil { + return err + } + + err = headerTable.Put(index, byt) + if err != nil { + return err + } + + // Update latest header + return latestHeaderTable.Put([]byte(""), index) +} + +// GetHeaderFromHeight will get a header given it's block height +func (c *Chaindb) GetHeaderFromHeight(index []byte) (*payload.BlockBase, error) { + headerTable := database.NewTable(c.db, HEADER) + hdrBytes, err := headerTable.Get(index) + if err != nil { + return nil, err + } + reader := bytes.NewReader(hdrBytes) + + blockBase := &payload.BlockBase{} + err = blockBase.Decode(reader) + if err != nil { + return nil, err + } + return blockBase, nil +} + +// GetLastHeader will get the header which was saved last in the database +func (c *Chaindb) GetLastHeader() (*payload.BlockBase, error) { + + latestHeaderTable := database.NewTable(c.db, LATESTHEADER) + index, err := latestHeaderTable.Get([]byte("")) + if err != nil { + return nil, err + } + return c.GetHeaderFromHeight(index) +} + +// GetBlockFromHash will return a block given it's hash +func (c *Chaindb) GetBlockFromHash(blockHash []byte) (*payload.Block, error) { + + blockTxTable := database.NewTable(c.db, BLOCKHASHTX) + + // To get a block we need to fetch: + // The transactions (1) + // The header (2) + + // Reconstruct block by fetching it's txs (1) + var txs []transaction.Transactioner + + // Get all Txhashes for this block + txHashes, err := blockTxTable.Prefix(blockHash) + if err != nil { + return nil, err + } + + // Get all Tx's given their hash + txTable := database.NewTable(c.db, TX) + for _, txHash := range txHashes { + + // Fetch tx by it's hash + txBytes, err := txTable.Get(txHash) + if err != nil { + return nil, err + } + reader := bufio.NewReader(bytes.NewReader(txBytes)) + + tx, err := transaction.FromReader(reader) + if err != nil { + return nil, err + } + txs = append(txs, tx) + } + + // Now fetch the header (2) + // We have the block hash, but headers are stored with their `Height` as key. + // We first search the `BlockHashHeight` table to get the height. + //Then we search the headers table with the height + hashHeightTable := database.NewTable(c.db, BLOCKHASHHEIGHT) + height, err := hashHeightTable.Get(blockHash) + if err != nil { + return nil, err + } + hdr, err := c.GetHeaderFromHeight(height) + if err != nil { + return nil, err + } + + // Construct block + block := &payload.Block{ + BlockBase: *hdr, + Txs: txs, + } + return block, nil +} + +// GetLastBlock will return the last block that has been saved +func (c *Chaindb) GetLastBlock() (*payload.Block, error) { + + latestBlockTable := database.NewTable(c.db, LATESTBLOCK) + blockHash, err := latestBlockTable.Get([]byte("")) + if err != nil { + return nil, err + } + return c.GetBlockFromHash(blockHash) +} + +func uint16ToBytes(x uint16) []byte { + index := make([]byte, 2) + binary.BigEndian.PutUint16(index, x) + return index +} + +func uint32ToBytes(x uint32) []byte { + index := make([]byte, 4) + binary.BigEndian.PutUint32(index, x) + return index +} + +func bytesConcat(args ...[]byte) []byte { + var res []byte + for _, arg := range args { + res = append(res, arg...) + } + return res +} diff --git a/pkg/chain/chaindb_test.go b/pkg/chain/chaindb_test.go new file mode 100644 index 000000000..b5656d30b --- /dev/null +++ b/pkg/chain/chaindb_test.go @@ -0,0 +1,201 @@ +package chain + +import ( + "bytes" + "math/rand" + "os" + "testing" + "time" + + "github.com/stretchr/testify/assert" + + "github.com/CityOfZion/neo-go/pkg/database" + "github.com/CityOfZion/neo-go/pkg/wire/payload" + "github.com/CityOfZion/neo-go/pkg/wire/payload/transaction" + "github.com/CityOfZion/neo-go/pkg/wire/util" +) + +var s = rand.NewSource(time.Now().UnixNano()) +var r = rand.New(s) + +func TestLastHeader(t *testing.T) { + _, cdb, hdrs := saveRandomHeaders(t) + + // Select last header from list of headers + lastHeader := hdrs[len(hdrs)-1] + // GetLastHeader from the database + hdr, err := cdb.GetLastHeader() + assert.Nil(t, err) + assert.Equal(t, hdr.Index, lastHeader.Index) + + // Clean up + os.RemoveAll(database.DbDir) +} + +func TestSaveHeader(t *testing.T) { + // save headers then fetch a random element + + db, _, hdrs := saveRandomHeaders(t) + + headerTable := database.NewTable(db, HEADER) + // check that each header was saved + for _, hdr := range hdrs { + index := uint32ToBytes(hdr.Index) + ok, err := headerTable.Has(index) + assert.Nil(t, err) + assert.True(t, ok) + } + + // Clean up + os.RemoveAll(database.DbDir) +} + +func TestSaveBlock(t *testing.T) { + + // Init databases + db, err := database.New("temp.test") + assert.Nil(t, err) + + cdb := &Chaindb{db} + + // Construct block0 and block1 + block0, block1 := twoBlocksLinked(t) + + // Save genesis header + err = cdb.saveHeader(&block0.BlockBase) + assert.Nil(t, err) + + // Save genesis block + err = cdb.saveBlock(block0, true) + assert.Nil(t, err) + + // Test genesis block saved + testBlockWasSaved(t, cdb, block0) + + // Save block1 header + err = cdb.saveHeader(&block1.BlockBase) + assert.Nil(t, err) + + // Save block1 + err = cdb.saveBlock(block1, false) + assert.Nil(t, err) + + // Test block1 was saved + testBlockWasSaved(t, cdb, block1) + + // Clean up + os.RemoveAll(database.DbDir) +} + +func testBlockWasSaved(t *testing.T, cdb *Chaindb, block payload.Block) { + // Fetch last block from database + lastBlock, err := cdb.GetLastBlock() + assert.Nil(t, err) + + // Get byte representation of last block from database + byts, err := lastBlock.Bytes() + assert.Nil(t, err) + + // Get byte representation of block that we saved + blockBytes, err := block.Bytes() + assert.Nil(t, err) + + // Should be equal + assert.True(t, bytes.Equal(byts, blockBytes)) +} + +func randomHeaders(t *testing.T) []*payload.BlockBase { + assert := assert.New(t) + hdrsMsg, err := payload.NewHeadersMessage() + assert.Nil(err) + + for i := 0; i < 2000; i++ { + err = hdrsMsg.AddHeader(randomBlockBase(t)) + assert.Nil(err) + } + + return hdrsMsg.Headers +} + +func randomBlockBase(t *testing.T) *payload.BlockBase { + + base := &payload.BlockBase{ + Version: r.Uint32(), + PrevHash: randUint256(t), + MerkleRoot: randUint256(t), + Timestamp: r.Uint32(), + Index: r.Uint32(), + ConsensusData: r.Uint64(), + NextConsensus: randUint160(t), + Witness: transaction.Witness{ + InvocationScript: []byte{0, 1, 2, 34, 56}, + VerificationScript: []byte{0, 12, 3, 45, 66}, + }, + Hash: randUint256(t), + } + return base +} + +func randomTxs(t *testing.T) []transaction.Transactioner { + + var txs []transaction.Transactioner + for i := 0; i < 10; i++ { + tx := transaction.NewContract(0) + tx.AddInput(transaction.NewInput(randUint256(t), uint16(r.Int()))) + tx.AddOutput(transaction.NewOutput(randUint256(t), r.Int63(), randUint160(t))) + txs = append(txs, tx) + } + return txs +} + +func saveRandomHeaders(t *testing.T) (database.Database, *Chaindb, []*payload.BlockBase) { + db, err := database.New("temp.test") + assert.Nil(t, err) + + cdb := &Chaindb{db} + + hdrs := randomHeaders(t) + + err = cdb.saveHeaders(hdrs) + assert.Nil(t, err) + return db, cdb, hdrs +} + +func randUint256(t *testing.T) util.Uint256 { + slice := make([]byte, 32) + _, err := r.Read(slice) + u, err := util.Uint256DecodeBytes(slice) + assert.Nil(t, err) + return u +} +func randUint160(t *testing.T) util.Uint160 { + slice := make([]byte, 20) + _, err := r.Read(slice) + u, err := util.Uint160DecodeBytes(slice) + assert.Nil(t, err) + return u +} + +// twoBlocksLinked will return two blocks, the second block spends from the utxos in the first +func twoBlocksLinked(t *testing.T) (payload.Block, payload.Block) { + genesisBase := randomBlockBase(t) + genesisTxs := randomTxs(t) + genesisBlock := payload.Block{BlockBase: *genesisBase, Txs: genesisTxs} + + var txs []transaction.Transactioner + + // Form transactions that spend from the genesis block + for _, tx := range genesisTxs { + txHash, err := tx.ID() + assert.Nil(t, err) + newTx := transaction.NewContract(0) + newTx.AddInput(transaction.NewInput(txHash, 0)) + newTx.AddOutput(transaction.NewOutput(randUint256(t), r.Int63(), randUint160(t))) + txs = append(txs, newTx) + } + + nextBase := randomBlockBase(t) + nextBlock := payload.Block{BlockBase: *nextBase, Txs: txs} + + return genesisBlock, nextBlock +} diff --git a/pkg/database/leveldb.go b/pkg/database/leveldb.go index b90ffaf65..a039c6010 100644 --- a/pkg/database/leveldb.go +++ b/pkg/database/leveldb.go @@ -3,14 +3,22 @@ package database import ( "github.com/syndtr/goleveldb/leveldb" "github.com/syndtr/goleveldb/leveldb/errors" + ldbutil "github.com/syndtr/goleveldb/leveldb/util" ) +//DbDir is the folder which all database files will be put under +// Structure /DbDir/net +const DbDir = "db/" + // LDB represents a leveldb object type LDB struct { db *leveldb.DB - path string + Path string } +// ErrNotFound means that the value was not found in the db +var ErrNotFound = errors.New("value not found for that key") + // Database contains all methods needed for an object to be a database type Database interface { // Has checks whether the key is in the database @@ -21,25 +29,30 @@ type Database interface { Get(key []byte) ([]byte, error) // Delete deletes the given value for the key from the database Delete(key []byte) error + //Prefix returns all values that start with key + Prefix(key []byte) ([][]byte, error) // Close closes the underlying db object Close() error } // New will return a new leveldb instance -func New(path string) *LDB { - db, err := leveldb.OpenFile(path, nil) +func New(path string) (*LDB, error) { + dbPath := DbDir + path + db, err := leveldb.OpenFile(dbPath, nil) if err != nil { - return nil + return nil, err } - if _, corrupted := err.(*errors.ErrCorrupted); corrupted { db, err = leveldb.RecoverFile(path, nil) + if err != nil { + return nil, err + } } return &LDB{ db, - path, - } + dbPath, + }, nil } // Has implements the database interface @@ -54,7 +67,15 @@ func (l *LDB) Put(key []byte, value []byte) error { // Get implements the database interface func (l *LDB) Get(key []byte) ([]byte, error) { - return l.db.Get(key, nil) + val, err := l.db.Get(key, nil) + if err == nil { + return val, nil + } + if err == leveldb.ErrNotFound { + return val, ErrNotFound + } + return val, err + } // Delete implements the database interface @@ -66,3 +87,28 @@ func (l *LDB) Delete(key []byte) error { func (l *LDB) Close() error { return l.db.Close() } + +// Prefix implements the database interface +func (l *LDB) Prefix(key []byte) ([][]byte, error) { + + var results [][]byte + + iter := l.db.NewIterator(ldbutil.BytesPrefix(key), nil) + for iter.Next() { + + value := iter.Value() + + // Copy the data, as we cannot modify it + // Once the iter has been released + deref := make([]byte, len(value)) + + copy(deref, value) + + // Append result + results = append(results, deref) + + } + iter.Release() + err := iter.Error() + return results, err +} diff --git a/pkg/database/leveldb_test.go b/pkg/database/leveldb_test.go index 0991682ee..61c831bd3 100644 --- a/pkg/database/leveldb_test.go +++ b/pkg/database/leveldb_test.go @@ -6,27 +6,31 @@ import ( "github.com/CityOfZion/neo-go/pkg/database" "github.com/stretchr/testify/assert" - "github.com/syndtr/goleveldb/leveldb/errors" ) const path = "temp" func cleanup(db *database.LDB) { db.Close() - os.RemoveAll(path) + os.RemoveAll(database.DbDir) } func TestDBCreate(t *testing.T) { - db := database.New(path) + + db, err := database.New(path) + assert.Nil(t, err) + assert.NotEqual(t, nil, db) cleanup(db) } func TestPutGet(t *testing.T) { - db := database.New(path) + + db, err := database.New(path) + assert.Nil(t, err) key := []byte("Hello") value := []byte("World") - err := db.Put(key, value) + err = db.Put(key, value) assert.Equal(t, nil, err) res, err := db.Get(key) @@ -36,25 +40,28 @@ func TestPutGet(t *testing.T) { } func TestPutDelete(t *testing.T) { - db := database.New(path) + db, err := database.New(path) + assert.Nil(t, err) key := []byte("Hello") value := []byte("World") - err := db.Put(key, value) + err = db.Put(key, value) err = db.Delete(key) assert.Equal(t, nil, err) res, err := db.Get(key) - assert.Equal(t, errors.ErrNotFound, err) + assert.Equal(t, database.ErrNotFound, err) assert.Equal(t, res, []byte{}) cleanup(db) } func TestHas(t *testing.T) { - db := database.New("temp") + + db, err := database.New(path) + assert.Nil(t, err) res, err := db.Has([]byte("NotExist")) assert.Equal(t, res, false) @@ -73,8 +80,12 @@ func TestHas(t *testing.T) { } func TestDBClose(t *testing.T) { - db := database.New("temp") - err := db.Close() + + db, err := database.New(path) + assert.Nil(t, err) + + err = db.Close() assert.Equal(t, nil, err) + cleanup(db) } diff --git a/pkg/database/table.go b/pkg/database/table.go index c36b80185..8c4cf3023 100644 --- a/pkg/database/table.go +++ b/pkg/database/table.go @@ -16,29 +16,35 @@ func NewTable(db Database, prefix []byte) *Table { // Has implements the database interface func (t *Table) Has(key []byte) (bool, error) { - key = append(t.prefix, key...) - return t.db.Has(key) + prefixedKey := append(t.prefix, key...) + return t.db.Has(prefixedKey) } // Put implements the database interface func (t *Table) Put(key []byte, value []byte) error { - key = append(t.prefix, key...) - return t.db.Put(key, value) + prefixedKey := append(t.prefix, key...) + return t.db.Put(prefixedKey, value) } // Get implements the database interface func (t *Table) Get(key []byte) ([]byte, error) { - key = append(t.prefix, key...) - return t.db.Get(key) + prefixedKey := append(t.prefix, key...) + return t.db.Get(prefixedKey) } // Delete implements the database interface func (t *Table) Delete(key []byte) error { - key = append(t.prefix, key...) - return t.db.Delete(key) + prefixedKey := append(t.prefix, key...) + return t.db.Delete(prefixedKey) } // Close implements the database interface func (t *Table) Close() error { return nil } + +// Prefix implements the database interface +func (t *Table) Prefix(key []byte) ([][]byte, error) { + prefixedKey := append(t.prefix, key...) + return t.db.Prefix(prefixedKey) +} diff --git a/pkg/peer/peer.go b/pkg/peer/peer.go index 6a87a4aa6..035dad835 100644 --- a/pkg/peer/peer.go +++ b/pkg/peer/peer.go @@ -195,13 +195,15 @@ func (p *Peer) PingLoop() { /*not implemented in other neo clients*/ } func (p *Peer) Run() error { err := p.Handshake() - + if err != nil { + return err + } go p.StartProtocol() go p.ReadLoop() go p.WriteLoop() //go p.PingLoop() // since it is not implemented. It will disconnect all other impls. - return err + return nil } diff --git a/pkg/wire/protocol/protocol.go b/pkg/wire/protocol/protocol.go index ebc916e50..d8b223267 100644 --- a/pkg/wire/protocol/protocol.go +++ b/pkg/wire/protocol/protocol.go @@ -30,3 +30,15 @@ const ( MainNet Magic = 7630401 TestNet Magic = 0x74746e41 ) + +// String implements the stringer interface +func (m Magic) String() string { + switch m { + case MainNet: + return "Mainnet" + case TestNet: + return "Testnet" + default: + return "UnknownNet" + } +} From 7d84d44b08a4b36442b2e6129baa4030c6cabe12 Mon Sep 17 00:00:00 2001 From: decentralisedkev <37423678+decentralisedkev@users.noreply.github.com> Date: Sat, 23 Mar 2019 19:09:25 +0000 Subject: [PATCH 07/24] [chain] (#209) - Add basic skeleton for chain struct --- pkg/chain/chain.go | 91 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 91 insertions(+) create mode 100644 pkg/chain/chain.go diff --git a/pkg/chain/chain.go b/pkg/chain/chain.go new file mode 100644 index 000000000..c1492cb8f --- /dev/null +++ b/pkg/chain/chain.go @@ -0,0 +1,91 @@ +package chain + +import ( + "errors" + + "github.com/CityOfZion/neo-go/pkg/wire/payload/transaction" + + "github.com/CityOfZion/neo-go/pkg/database" + "github.com/CityOfZion/neo-go/pkg/wire/payload" +) + +var ( + // ErrBlockAlreadyExists happens when you try to save the same block twice + ErrBlockAlreadyExists = errors.New("this block has already been saved in the database") +) + +// Chain represents a blockchain instance +type Chain struct { + db *Chaindb +} + +//New returns a new chain instance +func New(db database.Database) *Chain { + return &Chain{ + db: &Chaindb{db}, + } +} + +// SaveBlock verifies and saves the block in the database +// XXX: for now we will just save without verifying the block +// This function is called by the server and if an error is returned then +// the server informs the sync manager to redownload the block +// XXX:We should also check if the header is already saved in the database +// If not, then we need to validate the header with the rest of the chain +// For now we re-save the header +func (c *Chain) SaveBlock(msg payload.BlockMessage) error { + err := c.VerifyBlock(msg.Block) + if err != nil { + return err + } + //XXX(Issue): We can either check the hash here for genesisblock. + //We most likely will have it anyways after validation/ We can return it from VerifyBlock + // Or we can do it somewhere in startup, performance benefits + // won't be that big since it's just a bytes.Equal. + // so it's more about which is more readable and where it makes sense to put + return c.db.saveBlock(msg.Block, false) +} + +// VerifyBlock verifies whether a block is valid according +// to the rules of consensus +func (c *Chain) VerifyBlock(block payload.Block) error { + + // Check if we already have this block + // XXX: We can optimise by implementing a Has method + // caching the last block in memory + lastBlock, err := c.db.GetLastBlock() + if err != nil { + return err + } + // Check if we have already saved this block + // by looking if the latest block height is more than + // incoming block height + if lastBlock.Index > block.Index { + return ErrBlockAlreadyExists + } + + return nil +} + +// VerifyTx verifies whether a transaction is valid according +// to the rules of consensus +func (c *Chain) VerifyTx(tx transaction.Transactioner) error { + return nil +} + +// SaveHeaders will save the set of headers without validating +func (c *Chain) SaveHeaders(msg payload.HeadersMessage) error { + + err := c.verifyHeaders(msg.Headers) + if err != nil { + return err + } + return c.db.saveHeaders(msg.Headers) +} + +// verifyHeaders will be used to verify a batch of headers +// should only ever be called during the initial block download +// or when the node receives a HeadersMessage +func (c *Chain) verifyHeaders(hdrs []*payload.BlockBase) error { + return nil +} From beab4d186ff14aa372c65e1fe87cb1cd7524d358 Mon Sep 17 00:00:00 2001 From: dauTT <30392990+dauTT@users.noreply.github.com> Date: Mon, 25 Mar 2019 00:11:54 +0100 Subject: [PATCH 08/24] Fixed Txn Attribute econding/decoding (issue: #216) (#217) --- pkg/wire/payload/transaction/Attribute.go | 20 +++++------ .../payload/transaction/invocation_test.go | 34 +++++++++++++++++++ 2 files changed, 43 insertions(+), 11 deletions(-) diff --git a/pkg/wire/payload/transaction/Attribute.go b/pkg/wire/payload/transaction/Attribute.go index 7b747c1b0..0dc873da7 100644 --- a/pkg/wire/payload/transaction/Attribute.go +++ b/pkg/wire/payload/transaction/Attribute.go @@ -24,14 +24,13 @@ func (a *Attribute) Encode(bw *util.BinWriter) { } bw.Write(uint8(a.Usage)) - if a.Usage == DescriptionURL || a.Usage == Vote || (a.Usage >= Hash1 && a.Usage <= Hash15) { + if a.Usage == ContractHash || a.Usage == Vote || (a.Usage >= Hash1 && a.Usage <= Hash15) { bw.Write(a.Data[:32]) - - } else if a.Usage == Script { - bw.Write(a.Data[:20]) } else if a.Usage == ECDH02 || a.Usage == ECDH03 { bw.Write(a.Data[1:33]) - } else if a.Usage == CertURL || a.Usage == DescriptionURL || a.Usage == Description || a.Usage >= Remark { + } else if a.Usage == Script { + bw.Write(a.Data[:20]) + } else if a.Usage == DescriptionURL || a.Usage == Description || a.Usage >= Remark { bw.VarUint(uint64(len(a.Data))) bw.Write(a.Data) } else { @@ -43,17 +42,16 @@ func (a *Attribute) Encode(bw *util.BinWriter) { // Decode decodes the binary reader into an Attribute object func (a *Attribute) Decode(br *util.BinReader) { br.Read(&a.Usage) - if a.Usage == DescriptionURL || a.Usage == Vote || a.Usage >= Hash1 && a.Usage <= Hash15 { + if a.Usage == ContractHash || a.Usage == Vote || a.Usage >= Hash1 && a.Usage <= Hash15 { a.Data = make([]byte, 32) br.Read(&a.Data) - - } else if a.Usage == Script { - a.Data = make([]byte, 20) - br.Read(&a.Data) } else if a.Usage == ECDH02 || a.Usage == ECDH03 { a.Data = make([]byte, 32) br.Read(&a.Data) - } else if a.Usage == CertURL || a.Usage == DescriptionURL || a.Usage == Description || a.Usage >= Remark { + } else if a.Usage == Script { + a.Data = make([]byte, 20) + br.Read(&a.Data) + } else if a.Usage == DescriptionURL || a.Usage == Description || a.Usage >= Remark { lenData := br.VarUint() a.Data = make([]byte, lenData) br.Read(&a.Data) diff --git a/pkg/wire/payload/transaction/invocation_test.go b/pkg/wire/payload/transaction/invocation_test.go index fcd6a9818..c87ddc8d8 100644 --- a/pkg/wire/payload/transaction/invocation_test.go +++ b/pkg/wire/payload/transaction/invocation_test.go @@ -42,3 +42,37 @@ func TestEncodeDecodeInvoc(t *testing.T) { assert.Equal(t, nil, err) assert.Equal(t, rawtxBytes, buf.Bytes()) } + +func TestEncodeDecodeInvocAttributes(t *testing.T) { + // taken from mainnet cb0b5edc7e87b3b1bd9e029112fd3ce17c16d3de20c43ca1c0c26f3add578ecb + + rawtx := "d1015308005b950f5e010000140000000000000000000000000000000000000000141a1e29d6232d2148e1e71e30249835ea41eb7a3d53c1087472616e7366657267fb1c540417067c270dee32f21023aa8b9b71abce000000000000000002201a1e29d6232d2148e1e71e30249835ea41eb7a3d8110f9f504da6334935a2db42b18296d88700000014140461370f6847c4abbdddff54a3e1337e453ecc8133c882ec5b9aabcf0f47dafd3432d47e449f4efc77447ef03519b7808c450a998cca3ecc10e6536ed9db862ba23210285264b6f349f0fe86e9bb3044fde8f705b016593cf88cd5e8a802b78c7d2c950ac" + rawtxBytes, _ := hex.DecodeString(rawtx) + + i := NewInvocation(30) + + r := bytes.NewReader(rawtxBytes) + err := i.Decode(r) + assert.Equal(t, nil, err) + + assert.Equal(t, types.Invocation, i.Type) + + assert.Equal(t, 1, int(i.Version)) + + assert.Equal(t, 2, len(i.Attributes)) + + assert.Equal(t, Script, i.Attributes[0].Usage) + assert.Equal(t, "1a1e29d6232d2148e1e71e30249835ea41eb7a3d", hex.EncodeToString(i.Attributes[0].Data)) + assert.Equal(t, DescriptionURL, i.Attributes[1].Usage) + assert.Equal(t, "f9f504da6334935a2db42b18296d8870", hex.EncodeToString(i.Attributes[1].Data)) + + assert.Equal(t, "08005b950f5e010000140000000000000000000000000000000000000000141a1e29d6232d2148e1e71e30249835ea41eb7a3d53c1087472616e7366657267fb1c540417067c270dee32f21023aa8b9b71abce", hex.EncodeToString(i.Script)) + assert.Equal(t, "cb0b5edc7e87b3b1bd9e029112fd3ce17c16d3de20c43ca1c0c26f3add578ecb", i.Hash.ReverseString()) + + // Encode + buf := new(bytes.Buffer) + err = i.Encode(buf) + assert.Equal(t, nil, err) + assert.Equal(t, rawtxBytes, buf.Bytes()) + +} From ce1fe72607ad1f4bd8e04805c1c71511cb1e593d Mon Sep 17 00:00:00 2001 From: dauTT <30392990+dauTT@users.noreply.github.com> Date: Mon, 25 Mar 2019 02:04:54 +0100 Subject: [PATCH 09/24] Finalized size calculation methodology (#215) - Simplified Transactioner interface - Added size calculation test - Added utility methods in the address pkg: FromUint160, Uint160Decode --- pkg/chain/chaindb.go | 6 +- pkg/wire/payload/block_test.go | 92 ++++++++++++++++++++++++++++ pkg/wire/payload/transaction/base.go | 21 ++----- pkg/wire/util/address/address.go | 19 ++++++ 4 files changed, 118 insertions(+), 20 deletions(-) diff --git a/pkg/chain/chaindb.go b/pkg/chain/chaindb.go index 9b16b366a..5f1a7124e 100644 --- a/pkg/chain/chaindb.go +++ b/pkg/chain/chaindb.go @@ -98,7 +98,7 @@ func (c *Chaindb) saveTx(tx transaction.Transactioner, txIndex uint32, blockHash if err != nil { return err } - err = txTable.Put(txHash.Bytes(), tx.Bytes()) + err = txTable.Put(txHash.Bytes(), tx.BaseTx().Bytes()) if err != nil { return err } @@ -118,7 +118,7 @@ func (c *Chaindb) saveTx(tx transaction.Transactioner, txIndex uint32, blockHash // Save all of the utxos in a transaction // We do this additional save so that we can form a utxo database // and know when a transaction is a double spend. - utxos := tx.UTXOs() + utxos := tx.BaseTx().Outputs for utxoIndex, utxo := range utxos { err := c.saveUTXO(utxo, uint16(utxoIndex), txHash.Bytes(), blockHash) if err != nil { @@ -135,7 +135,7 @@ func (c *Chaindb) saveTx(tx transaction.Transactioner, txIndex uint32, blockHash // We do this so that once an output has been spent // It will be removed from the utxo database and cannot be spent again // If the output was never in the utxo database, this function will return an error - txos := tx.TXOs() + txos := tx.BaseTx().Inputs for _, txo := range txos { err := c.removeUTXO(txo) if err != nil { diff --git a/pkg/wire/payload/block_test.go b/pkg/wire/payload/block_test.go index 88c479c81..0736ff90d 100644 --- a/pkg/wire/payload/block_test.go +++ b/pkg/wire/payload/block_test.go @@ -5,6 +5,8 @@ import ( "encoding/hex" "testing" + "github.com/CityOfZion/neo-go/pkg/wire/util/address" + "github.com/CityOfZion/neo-go/pkg/wire/payload/transaction" "github.com/stretchr/testify/assert" ) @@ -78,3 +80,93 @@ func TestBlockDecodeEncode(t *testing.T) { assert.Equal(t, rawtx, hex.EncodeToString(buf.Bytes())) } +func TestBlockSizeCalculation(t *testing.T) { + // block taken from mainnet: 0006d3ff96e269f599eb1b5c5a527c218439e498dcc65b63794591bbcdc0516b + // The Size in golang is given by counting the number of bytes of an object. (len(Bytes)) + // its implementation is different from the corresponding C# and python implentation. But the result should + // should be the same.In this test we provide more details then necessary because in case of failure we can easily debug the + // root cause of the size calculation missmatch. + + rawBlock := "00000000ba33df12e8adbf38b6039e79ee91fdb8b1519e2e6154cb59c0653c81769288f4a22492109b7a84077ed7226c28612eb61428ea9ded9bdc952cdfc13deb4172ef85d1115b0bb62300abff35093d19a14a59e75d652b5d3827bf04c165bbe9ef95cca4bf5501fd45014012afae6df64195041e4764b57caa9e27fc2cfc596833163904136ec95816d104b44b3737d0e9f6b1b4445cd3b6a5cc80f6b0935675bc44dba44415eb309832b3404dc95bcf85e4635556a1d618e4ce947b26972992ed74788df5f9501b850ac0b40b7112d1ff30e4ade00369e16f0d13932d1ba76725e7682db072f8e2cd7752b840d12bb7dd45dd3b0e2098db5c67b6de55b7c40164937491fcaca1239b25860251224ead23ab232add78ccccd347239eae50ffc98f50b2a84c60ec5c3d284647a7406fabf6ca241b759af6b71080c0dfad7395632e989226a7e52f8cd2c133aeb2226e6e1aea47666fd81f578405a9f9bbd9d0bc523c3a44d7a5099ddc649feabe5f406188b8ee478731a89beeb76fdbd108eb0071b8f2b8678f40c5a1f387a491314336783255dee8cc5af4bf914dfeaacecc318fc13e02262658e39e8ce0631941b1f1552102486fd15702c4490a26703112a5cc1d0923fd697a33406bd5a1c00e0013b09a7021024c7b7fb6c310fccf1ba33b082519d82964ea93868d676662d4a59ad548df0e7d2102aaec38470f6aad0042c6e877cfd8087d2676b0f516fddd362801b9bd3936399e2103b209fd4f53a7170ea4444e0cb0a6bb6a53c2bd016926989cf85f9b0fba17a70c2103b8d9d5771d8f513aa0869b9cc8d50986403b78c6da36890638c3d46a5adce04a2102ca0e27697b9c248f6f16e085fd0061e26f44da85b58ee835c110caa5ec3ba5542102df48f60e8f3e01c48ff40b9b7f1310d7a8b2a193188befe1c2e3df740e89509357ae140000abff350900000000d101530800e1f5050000000014d8dd86f6d91eb2add2f2d8afeda2184ed94ac33314288017f85d80b889fe02beb5ff203ed9ef538f1653c1087472616e7366657267cf9472821400ceb06ca780c2a937fec5bbec51b900000000000000000220288017f85d80b889fe02beb5ff203ed9ef538f16f0153135323738393433373039313963623362646632650000014140b61b1a8d220c28633fa1a43ef02d334731b16013778664bc28db838d9ab8cfa64f9134fe952ad8a8eb8a6dd9d864055301bcdba4177fd6ee0f52b3f096db2fe4232102bca5c56af0f11e7042f5eaf3d8b2767feb3a8e3ba5668b00e6ea21cb7a215689ac8000000177de54907d16326ff29c3fcd4892afae32043e87ca844929157857093632c4010000019b7cffdaa674beae0f930ebe6085af9093e5fe56b34a5c220ccdcf6efc336fc500e9a435000000000849ee84e1f3daaf226589c05de2bf8bcae8c94001414092ba4569663dacd95921318658f8f40662bcff61fdcbbe08da0938a6c93a6d1075b76c86fa5454ca41f762d3b955b8d6755b79ccaf52754169b69e8904f166f8232102e019359f675526fc8505198647e31ed3044ccb0e5cc2ea22fb3bed5420cdf687ac800001ff076e656f2d6f6e6501f3617958f115913192d1de50c8c5ac2c411a4177c7801554d2bf69c27dac86c7010002e72d286979ee6cb1b7e65dfddfb2e384100b8d148e7758de42e4168b71792c606400000000000000915fe29d2d3847bce516f31fcd33f0fb1d90573be72d286979ee6cb1b7e65dfddfb2e384100b8d148e7758de42e4168b71792c6034ed1a0000000000c85c8dca1fbb7473522109cecaf3acac2e27afc60141406a5ad8c2b6e3783184703d22f3bee39a8b0b6f81477bff61833090361cd52ca5160d66215e6044c31aaed44304a28273c65c9ba736cc75341b45fb18995a6c922321039eae6f12690848807983df6accc1b2929de8582e772be6b3ac084a02a576272eacd10187084d3198000000000008660ceb7bc700000014b3a766ac60afa2990d9251db08138fd1facf07ed0800e3598f01000000209b7cffdaa674beae0f930ebe6085af9093e5fe56b34a5c220ccdcf6efc336fc514e420bbe1e5bcdfa1b43e97a57380a6cd8fddfd4c56c1096d616b654f6666657267bd097b2fcf70e1fd30a5c3ef51e662feeafeba0100000000000000000120e420bbe1e5bcdfa1b43e97a57380a6cd8fddfd4c01659c1aa6f2f42f7d0548a9680df197d4dda767434490e8bed3e25dbbac5206cd000001e72d286979ee6cb1b7e65dfddfb2e384100b8d148e7758de42e4168b71792c6001000000000000007335f929546270b8f811a0f9427b5712457107e70241400f871fcc8bb58d110ea7b1a4c34390542a925203b147bd68137c9edc79044a657cd5dc00514270d65a77a97c814d1f5090054b6c35f1cde698f765538ccf290c2321023056f0a219758bcd503ff4123a589962003331cb9e14168d649ae7426e3ec26eac4140a4036b311b31e0620fcde6e83ed29e6c7e7fcadf59dcac1ccae352a83608c8117dcd36ca64d5a2ce041b2274481ce65b6d30b463d1ca87a0269177d034ad218d232102a1e6ed9a5cff73ad33b7896465af8e9206eab9c8c75502868b783deb64f232eaacd10179207dcc40f47af5b26c52342d5292eb741b7beaa0d58a9a4a1563441fd0f40c30e1349b7cffdaa674beae0f930ebe6085af9093e5fe56b34a5c220ccdcf6efc336fc5b3a766ac60afa2990d9251db08138fd1facf07ed52c10b63616e63656c4f6666657267bd097b2fcf70e1fd30a5c3ef51e662feeafeba01000000000000000001202e5596a5c23eff8907aa180201c5a6f53c041dd101a9822ade57f6f9b6e999be06961af808137f3c9051b6c674bcc9e8aec8c1b6f1000001e72d286979ee6cb1b7e65dfddfb2e384100b8d148e7758de42e4168b71792c6001000000000000007335f929546270b8f811a0f9427b5712457107e7024140b5b58794f8c4f2493a2cff25aa7546a79b6f9e1f279911b2d15a07926084705aeaf64d18454f40feda8ddb9096745eb2e16a208320e9122007fac350892cb3d823210308ec2156f3366339c54c59e4c0342888665abdd76ccbd6b2020961225ccfa3f4ac41404f947ca69b7a2ec0926989f30f3d3d986d488b7c01270eee56326b2f0be856353448062a5d2d73ce34bfdacec3af42b082f1c5186fc063b38c321c1a1eb37157232102a1e6ed9a5cff73ad33b7896465af8e9206eab9c8c75502868b783deb64f232eaacd101792084e25c75cef1e92d39333408dae6e31799d6316b2d908aa094d0dc18f137484c349b7cffdaa674beae0f930ebe6085af9093e5fe56b34a5c220ccdcf6efc336fc5b3a766ac60afa2990d9251db08138fd1facf07ed52c10b63616e63656c4f6666657267bd097b2fcf70e1fd30a5c3ef51e662feeafeba01000000000000000001202e5596a5c23eff8907aa180201c5a6f53c041dd101ead2059c2cf4101cb02a1dae5875dc993189d047a84f05c080318779453b3c0e000001e72d286979ee6cb1b7e65dfddfb2e384100b8d148e7758de42e4168b71792c6001000000000000007335f929546270b8f811a0f9427b5712457107e7024140c4c924181d2b1516467f20fd3cf4b75ce5dd75d916318b77cd9ebd4bf9a3fb3bef47151adb01e0828bc127db6859f11890f86bc5bae2980cba5fcefba0d7edf523210308ec2156f3366339c54c59e4c0342888665abdd76ccbd6b2020961225ccfa3f4ac41407beaabc59bbc4db241aca812b624befcc595d3f8299e996a7d660decd3cb162afc6d810d5e3001aae5874c42b7031b1445f8562c209a2e2726ca1316dffd8867232102a1e6ed9a5cff73ad33b7896465af8e9206eab9c8c75502868b783deb64f232eaacd101530800e1f5050000000014d8dd86f6d91eb2add2f2d8afeda2184ed94ac33314288017f85d80b889fe02beb5ff203ed9ef538f1653c1087472616e7366657267f91d6b7085db7c5aaf09f19eeec1ca3c0db2c6ec00000000000000000220288017f85d80b889fe02beb5ff203ed9ef538f16f015313532373839343337373637356464383030353031000001414061609a0460a3ccfe1a9cb5db9f75811e08d52328f291a1b848ec607718be0a37206a90e3a81908c4b71ec859b684e493c088e640b2e2d471bd370aae50bdf160232102bca5c56af0f11e7042f5eaf3d8b2767feb3a8e3ba5668b00e6ea21cb7a215689ac80000001128218ba3c40a03066c862b0eaad5c06a39a1ae653e08f69e70f528ec4e18dc60500019b7cffdaa674beae0f930ebe6085af9093e5fe56b34a5c220ccdcf6efc336fc500a3e11100000000db2395d79fbd27d7b93b62cccd0fe0afc15ff80b0141404d953b03a1f53911fb1524a9f12126790e5e8468c05ed24ac3f71b881623510fcbe2f5497a4e8d1f1d1748774b9b3487be5f45dbd91bfb697a27ae2d2f2d9bb32321030ab39b99d8675cd9bd90aaec37cba964297cc817078d33e508ab11f1d245c068acd101792017a323cf2b0c650243be29d1b2c3ca69c141fdd7d72a36242c225f175c818a46349b7cffdaa674beae0f930ebe6085af9093e5fe56b34a5c220ccdcf6efc336fc5b3a766ac60afa2990d9251db08138fd1facf07ed52c10b63616e63656c4f6666657267bd097b2fcf70e1fd30a5c3ef51e662feeafeba01000000000000000001202e5596a5c23eff8907aa180201c5a6f53c041dd1010a190ead75c8235b2db69d9eb00040209ce719c3450d90bca893bcfcdbe5e58c000001e72d286979ee6cb1b7e65dfddfb2e384100b8d148e7758de42e4168b71792c6001000000000000007335f929546270b8f811a0f9427b5712457107e7024140b1dbd06a250ee76dc773e7efc915caea48732ca10f813d1027073dbba57032752c0ac7ed30fdb8b18df0abaf5831bd86e444785445253099b46c271676c4a16423210308ec2156f3366339c54c59e4c0342888665abdd76ccbd6b2020961225ccfa3f4ac4140a3b256ac1dd57de358bc1a45b632b3dd41f07d4c6d6e9ffe33fd0cf7c44e29673100bc8a8760727b46bf96642d6dd4a9c4edf657fb2a27b83f564a5b91408ad9232102a1e6ed9a5cff73ad33b7896465af8e9206eab9c8c75502868b783deb64f232eaacd1014e03a0860114d023de91710f63a0259e8d95d1f6563e1572783c14b18f53a903d7873b1453a748c3f80787eca2e30f53c1087472616e7366657267187fc13bec8ff0906c079e7f4cc8276709472913000000000000000004fee36e656f2d6f6e652d696e766f6b653a7b22636f6e7472616374223a22307831333239343730393637323763383463376639653037366339306630386665633362633137663138222c226d6574686f64223a227472616e73666572222c22706172616d73223a5b5b2266726f6d222c22307830666533613265633837303766386333343861373533313433623837643730336139353338666231225d2c5b22746f222c22307833633738373231353365353666366431393538643965323561303633306637313931646532336430225d2c5b2276616c7565222c22302e303031225d5d7dff076e656f2d6f6e65ff0a3233313635343031383420b18f53a903d7873b1453a748c3f80787eca2e30f0000014140910c4b3be37fe09052215f99ef546342c21ded22ef862c165ff0da3ff1087274d0fafb5bbac25aa927dc477b239df732d95afe782a46caeaca669ae096659f13232102c2dbc83931d5e550b95ceab8a94c6af37735fe2aa4e9fb217bce46001937b2f1acd1014e03a0860114de33c5d07f933c0f90da952cdfe2677d3f9a24d714f26e3a75379dc20c8642f3f11e20af76b12065f453c1087472616e7366657267aa67d0447c61bdddc4d1690d2269d772f9c37795000000000000000004fee36e656f2d6f6e652d696e766f6b653a7b22636f6e7472616374223a22307839353737633366393732643736393232306436396431633464646264363137633434643036376161222c226d6574686f64223a227472616e73666572222c22706172616d73223a5b5b2266726f6d222c22307866343635323062313736616632303165663166333432383630636332396433373735336136656632225d2c5b22746f222c22307864373234396133663764363765326466326339356461393030663363393337666430633533336465225d2c5b2276616c7565222c22302e303031225d5d7dff076e656f2d6f6e65ff0933363133393630393120f26e3a75379dc20c8642f3f11e20af76b12065f40000014140b1630c1546547df4b69fdf6c68fcbf9a3f1de2787d0ea7237aea639ab0a227a092147e47194595c0d960df0d209773da32808da767c15730926bcf844ed12f702321021a9f5ed87fe58e7a366c20975d4112698e4c0ccb3ba9cbce0400a482ecf99b67acd1014e03a08601148ba89ee5ab5c4975e0e12f88a8ce4aa8928108c9145e5c739e5d4b5a29af596c2cc162f1facded25fe53c1087472616e7366657267952d12a025325e56a4cb3ba2d469b1e23c7c77a0000000000000000004fee36e656f2d6f6e652d696e766f6b653a7b22636f6e7472616374223a22307861303737376333636532623136396434613233626362613435363565333232356130313232643935222c226d6574686f64223a227472616e73666572222c22706172616d73223a5b5b2266726f6d222c22307866653235656463646661663136326331326336633539616632393561346235643965373335633565225d2c5b22746f222c22307863393038383139326138346163656138383832666531653037353439356361626535396561383862225d2c5b2276616c7565222c22302e303031225d5d7dff076e656f2d6f6e65ff0a33383237333138333238205e5c739e5d4b5a29af596c2cc162f1facded25fe000001414050f556b29417eb44b66b283a72ff33b697f49b2b6d1520eb4fe5a9974632c1b1586b3e7f0aafac4707d65f89daa86592c8ad98c78016769a5de904330d02d2ed23210201008fe0ffcdab73b598c89c6ae2b46d90de38287abd7dd50a325d0bfb2469d5acd1014e03a086011444d65fda3f2062502c03c2bfd85c700a0d046fae149d12fcef2c830d73a570e9b89857962fcf3a619a53c1087472616e7366657267187fc13bec8ff0906c079e7f4cc8276709472913000000000000000004fee36e656f2d6f6e652d696e766f6b653a7b22636f6e7472616374223a22307831333239343730393637323763383463376639653037366339306630386665633362633137663138222c226d6574686f64223a227472616e73666572222c22706172616d73223a5b5b2266726f6d222c22307839613631336163663266393635373938623865393730613537333064383332636566666331323964225d2c5b22746f222c22307861653666303430643061373035636438626663323033326335303632323033666461356664363434225d2c5b2276616c7565222c22302e303031225d5d7dff076e656f2d6f6e65ff0a31353931373638373437209d12fcef2c830d73a570e9b89857962fcf3a619a00000141409ec506aab7045da733d02c8c9ebaa615d9aa02fa8c4d8eb35ad08a3b4843167cdd059094f492f77e68efa2b66d87305e13b3f08a326cf4b3dadaa29aaa64b30a23210275699ef1532219ee8e9d2d8a75b6b96b7cb4d5f9788391de31daceebed9ac8edacd1014e03a086011444bb85601b2d8c5247a1999dfb18ee7928e10cdf1479ed35e989051c8f2404b869c17ce75912de78b953c1087472616e7366657267aa67d0447c61bdddc4d1690d2269d772f9c37795000000000000000004fee36e656f2d6f6e652d696e766f6b653a7b22636f6e7472616374223a22307839353737633366393732643736393232306436396431633464646264363137633434643036376161222c226d6574686f64223a227472616e73666572222c22706172616d73223a5b5b2266726f6d222c22307862393738646531323539653737636331363962383034323438663163303538396539333565643739225d2c5b22746f222c22307864663063653132383739656531386662396439396131343735323863326431623630383562623434225d2c5b2276616c7565222c22302e303031225d5d7dff076e656f2d6f6e65ff0a333738313933373030382079ed35e989051c8f2404b869c17ce75912de78b9000001414081a2d21af431df3345478836ae24dc7714ce647daa1daac3a99ed17645a8c86a981d86c5d2280af45259a1cf4a85c115ecbe8ff5a398c0bea23c218c0c8c26f02321020be10b4bddffe752a7cfaa16d1718ce6da460608ca41ad9bf7ff66dc5f60c860acd10179201322db68df23cee8da600e7cf5875a8d919bf8e318bbe7afbcd1fdbd5758d67b349b7cffdaa674beae0f930ebe6085af9093e5fe56b34a5c220ccdcf6efc336fc5b3a766ac60afa2990d9251db08138fd1facf07ed52c10b63616e63656c4f6666657267bd097b2fcf70e1fd30a5c3ef51e662feeafeba01000000000000000001202e5596a5c23eff8907aa180201c5a6f53c041dd10181c3670591e333a19a14aae556d211f3490163380c1f28ef55b0bb0e4a8aec1a000001e72d286979ee6cb1b7e65dfddfb2e384100b8d148e7758de42e4168b71792c6001000000000000007335f929546270b8f811a0f9427b5712457107e7024140dd2aa6075329f43375655c83b04d343974fb13908888fa6b242b599c789c1cf635b530b6834093332720f93d185b450884951acfb261aec61fe9057efbdeb4b523210308ec2156f3366339c54c59e4c0342888665abdd76ccbd6b2020961225ccfa3f4ac4140f152dc9454ebf518bc0ab8192543add5844ad7007177986a3dd038db313740ea72abb5d49866caa4d470d75182a7e715fd2b9a6e345ef2568bf82e92878743db232102a1e6ed9a5cff73ad33b7896465af8e9206eab9c8c75502868b783deb64f232eaac8000000198f4ec54f46fd86c0955accb7bda7ff06ffc10c45651206fbcbfeabe78c597750000019b7cffdaa674beae0f930ebe6085af9093e5fe56b34a5c220ccdcf6efc336fc500e40b54020000002fb7e583c973498fef06f317a9762507e7ac0306014140c4a7388365a4e8a93e2229fd617805902773a8ef7354e95487819eec59379a6a8fb045ed126025db151336e2393eaf75dd524f1ebe872f92185f8d1a6a71a6a5232102b37eaec8631a5bb8579d4ba268e2fbc9c81b555f08558bba6eccb9d1448332c5ac02000155810e77f1dd89622915f3fceb051ac8abeaa2e7ea7a2c38e0744137c3c8982b0000000001e72d286979ee6cb1b7e65dfddfb2e384100b8d148e7758de42e4168b71792c605e1502000000000024fdf844d637cf5a60a9744f1ab8ab8d6f2c18cf0141407dafb244fceee305d477892a21ca8cbd31906fbcc233e0bfccd0fe91d4f6dafdd067f7301cd3e8c7c6197f71615df4b7cedb987eb3ebff79ad729bb8768e1e50232102f8a4a73deefad4c114039b854cc4e974c409f30720e2567d15dea2204b8c45f7acd1012000c108776974686472617769bd097b2fcf70e1fd30a5c3ef51e662feeafeba01000000000000000003a15100000000000000000000000000000000000000000000000000000000000000a232e125258b7db0a0dffde5bd03b2b859253538ab000000000000000000000000a48098c835c493eb3b1967e44150630cc6435e564e00000000000000000000000001dfeee6c60db6403b5538e86120f01e60dcea749c73ce36b9058f050be8b7d2c9000001e72d286979ee6cb1b7e65dfddfb2e384100b8d148e7758de42e4168b71792c600100000000000000bd097b2fcf70e1fd30a5c3ef51e662feeafeba010102000000d101530800e1f5050000000014d8dd86f6d91eb2add2f2d8afeda2184ed94ac33314288017f85d80b889fe02beb5ff203ed9ef538f1653c1087472616e7366657267f56c89be8bfcdec617e2402b5c3fd5b6d71b820d00000000000000000220288017f85d80b889fe02beb5ff203ed9ef538f16f0153135323738393433383433393264623331653662630000014140654af3a0cc69faa3dd42ff76c4012aa9c72e269bba004d6e910f195e33b2ecae980be4531a3677f27d3c90f4196632790997078bca4f8471c6db43b55928c3ef232102bca5c56af0f11e7042f5eaf3d8b2767feb3a8e3ba5668b00e6ea21cb7a215689ac" + rawBlockBytes, _ := hex.DecodeString(rawBlock) + + b := Block{} + + r := bytes.NewReader(rawBlockBytes) + b.Decode(r) + + expected := []struct { + ID string + Type string + Size int + Version int + InputsLen int + OutputsLen int + AttributesLen int + WitnessesLen int + }{ // 20 trans + {ID: "f59b04d8e6526684b94b5f8cdbdf691feaff5d45e9aa8e2325a668f1b9130786", Type: "MinerTransaction", Size: 10, Version: 0, InputsLen: 0, OutputsLen: 0, AttributesLen: 0, WitnessesLen: 0}, + {ID: "7463345f771e70019185d72fa5bd00fbb4f26735daae398ecc6540419332d81e", Type: "InvocationTransaction", Size: 244, Version: 1, InputsLen: 0, OutputsLen: 0, AttributesLen: 2, WitnessesLen: 1}, + {ID: "cf3aeda21d320ec9b49d322f2b88fea21aa7e9bf243c1e02dfe08f5cc82b74b0", Type: "ContractTransaction", Size: 202, Version: 0, InputsLen: 1, OutputsLen: 1, AttributesLen: 0, WitnessesLen: 1}, + {ID: "07e502d13ae6255cfabbc9ee2f78a48fc1c43a4f7f713f128342db721bc01af5", Type: "ContractTransaction", Size: 271, Version: 0, InputsLen: 1, OutputsLen: 2, AttributesLen: 1, WitnessesLen: 1}, + {ID: "0de0baf53136c188bdd179fed9530dfb7dd80697fd59e47ffe294db4f421eb67", Type: "InvocationTransaction", Size: 469, Version: 1, InputsLen: 1, OutputsLen: 1, AttributesLen: 1, WitnessesLen: 2}, + {ID: "233c8b00ab6a43aafae7fcc2be47fc46493185bb3376160b5809cb745aee3329", Type: "InvocationTransaction", Size: 455, Version: 1, InputsLen: 1, OutputsLen: 1, AttributesLen: 1, WitnessesLen: 2}, + {ID: "8bf3ae0c692fc830753029fcb6575625ea8181b444cffcbe38404a28b77b3856", Type: "InvocationTransaction", Size: 455, Version: 1, InputsLen: 1, OutputsLen: 1, AttributesLen: 1, WitnessesLen: 2}, + {ID: "c5285e1460191e1ca7fc07e3c26c5facebb033d56b63b7a41ebf11f2a1cb4306", Type: "InvocationTransaction", Size: 244, Version: 1, InputsLen: 0, OutputsLen: 0, AttributesLen: 2, WitnessesLen: 1}, + {ID: "dce8b5f6dc093a910e405a230e9b7d546688d411cf960c8a1cc7d386d89b56d6", Type: "ContractTransaction", Size: 202, Version: 0, InputsLen: 1, OutputsLen: 1, AttributesLen: 0, WitnessesLen: 1}, + {ID: "4cce087cadfa99c2adeaaf1916ada025db124cef8f05d4535b0ad8047ef7d29e", Type: "InvocationTransaction", Size: 455, Version: 1, InputsLen: 1, OutputsLen: 1, AttributesLen: 1, WitnessesLen: 2}, + {ID: "67df57a20c9d3b2942925f2c66fdc15a21be2c229a22122f6acbdac4dd10bf0a", Type: "InvocationTransaction", Size: 466, Version: 1, InputsLen: 0, OutputsLen: 0, AttributesLen: 4, WitnessesLen: 1}, + {ID: "ad51030b30e016293caed92781b3bb3f993f86c15ab1153582f658d603fe23db", Type: "InvocationTransaction", Size: 465, Version: 1, InputsLen: 0, OutputsLen: 0, AttributesLen: 4, WitnessesLen: 1}, + {ID: "1db2d62ad3530f1ae6ca7bd95e766beaff97058681f0e203d8744d7bba065012", Type: "InvocationTransaction", Size: 466, Version: 1, InputsLen: 0, OutputsLen: 0, AttributesLen: 4, WitnessesLen: 1}, + {ID: "7d7bb6f0db6a71aca85fc9267fa6a59654b00b5f778a39c27214c68f11950f61", Type: "InvocationTransaction", Size: 466, Version: 1, InputsLen: 0, OutputsLen: 0, AttributesLen: 4, WitnessesLen: 1}, + {ID: "1d534dcf1ce63a9ea9328eab309891ea2a0a5cb11e95cabf22860ee1fb649521", Type: "InvocationTransaction", Size: 466, Version: 1, InputsLen: 0, OutputsLen: 0, AttributesLen: 4, WitnessesLen: 1}, + {ID: "8f4c9089871a4ad0076b27c061395079e0862f685f27e4bc01b7bac67b0cf8d0", Type: "InvocationTransaction", Size: 455, Version: 1, InputsLen: 1, OutputsLen: 1, AttributesLen: 1, WitnessesLen: 2}, + {ID: "52b4653fca02e8042092456490036f0b9b18b339f65d9c334e7e9d2b4599f8db", Type: "ContractTransaction", Size: 202, Version: 0, InputsLen: 1, OutputsLen: 1, AttributesLen: 0, WitnessesLen: 1}, + {ID: "2b4854b1f46c9af0eb06587fd375355adfeea3c8f5921295421251af93d737e1", Type: "ClaimTransaction", Size: 203, Version: 0, InputsLen: 0, OutputsLen: 1, AttributesLen: 0, WitnessesLen: 1}, + {ID: "61e95e5b14625e897423670dfc3babf021d6b99ca2a73203dd1ac2604a2daadf", Type: "InvocationTransaction", Size: 244, Version: 1, InputsLen: 1, OutputsLen: 1, AttributesLen: 3, WitnessesLen: 1}, + {ID: "b361dfec8c2cde980b340d2c3ec63cecaea634f91b6d76f24a586aa60fbde483", Type: "InvocationTransaction", Size: 244, Version: 1, InputsLen: 0, OutputsLen: 0, AttributesLen: 2, WitnessesLen: 1}, + } + + for i, tx := range b.Txs { + txID, err := tx.ID() + assert.Equal(t, nil, err) + assert.Equal(t, expected[i].ID, txID.ReverseString()) + + assert.Equal(t, expected[i].Size, len(tx.BaseTx().Bytes())) + assert.Equal(t, expected[i].Type, tx.BaseTx().Type.String()) + assert.Equal(t, expected[i].Version, int(tx.BaseTx().Version)) + assert.Equal(t, expected[i].InputsLen, len(tx.BaseTx().Inputs)) + assert.Equal(t, expected[i].OutputsLen, len(tx.BaseTx().Outputs)) + assert.Equal(t, expected[i].AttributesLen, len(tx.BaseTx().Attributes)) + assert.Equal(t, expected[i].WitnessesLen, len(tx.BaseTx().Witnesses)) + } + + assert.Equal(t, len(expected), len(b.Txs)) + + // Block specific tests + assert.Equal(t, 0, int(b.Version)) + assert.Equal(t, "f4889276813c65c059cb54612e9e51b1b8fd91ee799e03b638bfade812df33ba", b.PrevHash.ReverseString()) + assert.Equal(t, "ef7241eb3dc1df2c95dc9bed9dea2814b62e61286c22d77e07847a9b109224a2", b.MerkleRoot.ReverseString()) + assert.Equal(t, 1527894405, int(b.Timestamp)) + assert.Equal(t, 2340363, int(b.Index)) + + nextConsensus, err := address.FromUint160(b.NextConsensus) + assert.Equal(t, nil, err) + assert.Equal(t, "APyEx5f4Zm4oCHwFWiSTaph1fPBxZacYVR", nextConsensus) + + assert.Equal(t, "4012afae6df64195041e4764b57caa9e27fc2cfc596833163904136ec95816d104b44b3737d0e9f6b1b4445cd3b6a5cc80f6b0935675bc44dba44415eb309832b3404dc95bcf85e4635556a1d618e4ce947b26972992ed74788df5f9501b850ac0b40b7112d1ff30e4ade00369e16f0d13932d1ba76725e7682db072f8e2cd7752b840d12bb7dd45dd3b0e2098db5c67b6de55b7c40164937491fcaca1239b25860251224ead23ab232add78ccccd347239eae50ffc98f50b2a84c60ec5c3d284647a7406fabf6ca241b759af6b71080c0dfad7395632e989226a7e52f8cd2c133aeb2226e6e1aea47666fd81f578405a9f9bbd9d0bc523c3a44d7a5099ddc649feabe5f406188b8ee478731a89beeb76fdbd108eb0071b8f2b8678f40c5a1f387a491314336783255dee8cc5af4bf914dfeaacecc318fc13e02262658e39e8ce0631941b1", hex.EncodeToString(b.Witness.InvocationScript)) + assert.Equal(t, "552102486fd15702c4490a26703112a5cc1d0923fd697a33406bd5a1c00e0013b09a7021024c7b7fb6c310fccf1ba33b082519d82964ea93868d676662d4a59ad548df0e7d2102aaec38470f6aad0042c6e877cfd8087d2676b0f516fddd362801b9bd3936399e2103b209fd4f53a7170ea4444e0cb0a6bb6a53c2bd016926989cf85f9b0fba17a70c2103b8d9d5771d8f513aa0869b9cc8d50986403b78c6da36890638c3d46a5adce04a2102ca0e27697b9c248f6f16e085fd0061e26f44da85b58ee835c110caa5ec3ba5542102df48f60e8f3e01c48ff40b9b7f1310d7a8b2a193188befe1c2e3df740e89509357ae", hex.EncodeToString(b.Witness.VerificationScript)) + assert.Equal(t, "0006d3ff96e269f599eb1b5c5a527c218439e498dcc65b63794591bbcdc0516b", b.Hash.ReverseString()) + + bb, err := b.Bytes() + assert.Equal(t, nil, err) + + // test size of the block + assert.Equal(t, 7360, len(bb)) + + buf := new(bytes.Buffer) + + b.Encode(buf) + + assert.Equal(t, rawBlock, hex.EncodeToString(buf.Bytes())) +} diff --git a/pkg/wire/payload/transaction/base.go b/pkg/wire/payload/transaction/base.go index 541e958e4..567e907d8 100644 --- a/pkg/wire/payload/transaction/base.go +++ b/pkg/wire/payload/transaction/base.go @@ -18,11 +18,8 @@ type decodeExclusiveFields func(br *util.BinReader) type Transactioner interface { Encode(w io.Writer) error Decode(r io.Reader) error + BaseTx() *Base ID() (util.Uint256, error) - Bytes() []byte - UTXOs() []*Output - TXOs() []*Input - Witness() []*Witness } // Base transaction is the template for all other transactions @@ -199,17 +196,7 @@ func (b *Base) Bytes() []byte { return buf.Bytes() } -// UTXOs returns the outputs in the tx -func (b *Base) UTXOs() []*Output { - return b.Outputs -} - -// TXOs returns the inputs in the tx -func (b *Base) TXOs() []*Input { - return b.Inputs -} - -// Witness returns the witnesses in the tx -func (b *Base) Witness() []*Witness { - return b.Witnesses +// BaseTx returns the Base object in a transaction +func (b *Base) BaseTx() *Base { + return b } diff --git a/pkg/wire/util/address/address.go b/pkg/wire/util/address/address.go index 84daf5345..12f09c63a 100644 --- a/pkg/wire/util/address/address.go +++ b/pkg/wire/util/address/address.go @@ -4,6 +4,7 @@ import ( "encoding/hex" "github.com/CityOfZion/neo-go/pkg/crypto/base58" + "github.com/CityOfZion/neo-go/pkg/wire/util" ) // ToScriptHash converts an address to a script hash @@ -17,3 +18,21 @@ func ToScriptHash(address string) string { scriptHash := (decodedAddressAsHex[2:42]) return scriptHash } + +// FromUint160 returns the "NEO address" from the given +// Uint160. +func FromUint160(u util.Uint160) (string, error) { + // Dont forget to prepend the Address version 0x17 (23) A + b := append([]byte{0x17}, u.Bytes()...) + return base58.CheckEncode(b) +} + +// Uint160Decode attempts to decode the given NEO address string +// into an Uint160. +func Uint160Decode(s string) (u util.Uint160, err error) { + b, err := base58.CheckDecode(s) + if err != nil { + return u, err + } + return util.Uint160DecodeBytes(b[1:21]) +} From dc5de1fa6db9655c11d31215e0c38c169fd418e9 Mon Sep 17 00:00:00 2001 From: decentralisedkev <37423678+decentralisedkev@users.noreply.github.com> Date: Thu, 28 Mar 2019 19:09:55 +0000 Subject: [PATCH 10/24] [Peer] Refactor (#240) [Peer] - Closes #239 - moved response handlers to their own functions - removed DefaultConfig from LocalConfig file - passed peer as a parameter to all response handlers - added peer start height - refactored NewPeer function to be more concise and clear - removed empty lines at end of functions - Added AddMessage/RemoveMessage for Detector in outgoing and ingoing requests for Block and Headers --- pkg/peer/config.go | 34 ++----- pkg/peer/peer.go | 167 ++++++----------------------------- pkg/peer/peer_test.go | 31 ++----- pkg/peer/responsehandlers.go | 111 +++++++++++++++++++++++ pkg/peer/stall/stall.go | 16 ++-- pkg/peer/stall/stall_test.go | 2 +- 6 files changed, 165 insertions(+), 196 deletions(-) create mode 100644 pkg/peer/responsehandlers.go diff --git a/pkg/peer/config.go b/pkg/peer/config.go index e4794c282..47c688741 100644 --- a/pkg/peer/config.go +++ b/pkg/peer/config.go @@ -14,34 +14,18 @@ type LocalConfig struct { ProtocolVer protocol.Version Relay bool Port uint16 - // pointer to config will keep the startheight updated for each version - //Message we plan to send - StartHeight func() uint32 + + // pointer to config will keep the startheight updated + StartHeight func() uint32 + + // Response Handlers OnHeader func(*Peer, *payload.HeadersMessage) - OnGetHeaders func(msg *payload.GetHeadersMessage) // returns HeaderMessage + OnGetHeaders func(*Peer, *payload.GetHeadersMessage) OnAddr func(*Peer, *payload.AddrMessage) OnGetAddr func(*Peer, *payload.GetAddrMessage) OnInv func(*Peer, *payload.InvMessage) - OnGetData func(msg *payload.GetDataMessage) + OnGetData func(*Peer, *payload.GetDataMessage) OnBlock func(*Peer, *payload.BlockMessage) - OnGetBlocks func(msg *payload.GetBlocksMessage) + OnGetBlocks func(*Peer, *payload.GetBlocksMessage) + OnTx func(*Peer, *payload.TXMessage) } - -// func DefaultConfig() LocalConfig { -// return LocalConfig{ -// Net: protocol.MainNet, -// UserAgent: "NEO-GO-Default", -// Services: protocol.NodePeerService, -// Nonce: 1200, -// ProtocolVer: 0, -// Relay: false, -// Port: 10332, -// // pointer to config will keep the startheight updated for each version -// //Message we plan to send -// StartHeight: DefaultHeight, -// } -// } - -// func DefaultHeight() uint32 { -// return 10 -// } diff --git a/pkg/peer/peer.go b/pkg/peer/peer.go index 035dad835..94f9c685c 100644 --- a/pkg/peer/peer.go +++ b/pkg/peer/peer.go @@ -58,6 +58,8 @@ type Peer struct { config LocalConfig conn net.Conn + startHeight uint32 + // atomic vals disconnected int32 @@ -84,20 +86,18 @@ type Peer struct { // NewPeer returns a new NEO peer func NewPeer(con net.Conn, inbound bool, cfg LocalConfig) *Peer { - p := Peer{} - p.inch = make(chan func(), inputBufferSize) - p.outch = make(chan func(), outputBufferSize) - p.quitch = make(chan struct{}, 1) - p.inbound = inbound - p.config = cfg - p.conn = con - p.createdAt = time.Now() - p.addr = p.conn.RemoteAddr().String() - - p.Detector = stall.NewDetector(responseTime, tickerInterval) - - // TODO: set the unchangeable states - return &p + return &Peer{ + inch: make(chan func(), inputBufferSize), + outch: make(chan func(), outputBufferSize), + quitch: make(chan struct{}, 1), + inbound: inbound, + config: cfg, + conn: con, + createdAt: time.Now(), + startHeight: 0, + addr: con.RemoteAddr().String(), + Detector: stall.NewDetector(responseTime, tickerInterval), + } } // Write to a peer @@ -125,7 +125,6 @@ func (p *Peer) Disconnect() { p.conn.Close() fmt.Println("Disconnected Peer with address", p.RemoteAddr().String()) - } // Port returns the peers port @@ -138,6 +137,11 @@ func (p *Peer) CreatedAt() time.Time { return p.createdAt } +// Height returns the latest recorded height of this peer +func (p *Peer) Height() uint32 { + return p.startHeight +} + // CanRelay returns true, if the peer can relay information func (p *Peer) CanRelay() bool { return p.relay @@ -163,11 +167,6 @@ func (p *Peer) Inbound() bool { return p.inbound } -// UserAgent returns this nodes, useragent -func (p *Peer) UserAgent() string { - return p.config.UserAgent -} - // IsVerackReceived returns true, if this node has // received a verack from this peer func (p *Peer) IsVerackReceived() bool { @@ -204,7 +203,6 @@ func (p *Peer) Run() error { //go p.PingLoop() // since it is not implemented. It will disconnect all other impls. return nil - } // StartProtocol run as a go-routine, will act as our queue for messages @@ -305,128 +303,17 @@ func (p *Peer) WriteLoop() { } } -// OnGetData is called when a GetData message is received -func (p *Peer) OnGetData(msg *payload.GetDataMessage) { - - p.inch <- func() { - if p.config.OnInv != nil { - p.config.OnGetData(msg) - } - fmt.Println("That was an getdata Message please pass func down through config", msg.Command()) - } -} - -//OnTX is callwed when a TX message is received -func (p *Peer) OnTX(msg *payload.TXMessage) { - - p.inch <- func() { - getdata, err := payload.NewGetDataMessage(payload.InvTypeTx) - if err != nil { - fmt.Println("Eor", err) - } - id, err := msg.Tx.ID() - getdata.AddHash(id) - p.Write(getdata) - } -} - -// OnInv is called when a Inv message is received -func (p *Peer) OnInv(msg *payload.InvMessage) { - - p.inch <- func() { - if p.config.OnInv != nil { - p.config.OnInv(p, msg) - } - fmt.Println("That was an inv Message please pass func down through config", msg.Command()) - } -} - -// OnGetHeaders is called when a GetHeaders message is received -func (p *Peer) OnGetHeaders(msg *payload.GetHeadersMessage) { - p.inch <- func() { - if p.config.OnGetHeaders != nil { - p.config.OnGetHeaders(msg) - } - fmt.Println("That was a getheaders message, please pass func down through config", msg.Command()) - - } -} - -// OnAddr is called when a Addr message is received -func (p *Peer) OnAddr(msg *payload.AddrMessage) { - p.inch <- func() { - if p.config.OnAddr != nil { - p.config.OnAddr(p, msg) - } - fmt.Println("That was a addr message, please pass func down through config", msg.Command()) - - } -} - -// OnGetAddr is called when a GetAddr message is received -func (p *Peer) OnGetAddr(msg *payload.GetAddrMessage) { - p.inch <- func() { - if p.config.OnGetAddr != nil { - p.config.OnGetAddr(p, msg) - } - fmt.Println("That was a getaddr message, please pass func down through config", msg.Command()) - - } -} - -// OnGetBlocks is called when a GetBlocks message is received -func (p *Peer) OnGetBlocks(msg *payload.GetBlocksMessage) { - p.inch <- func() { - if p.config.OnGetBlocks != nil { - p.config.OnGetBlocks(msg) - } - fmt.Println("That was a getblocks message, please pass func down through config", msg.Command()) - } -} - -// OnBlocks is called when a Blocks message is received -func (p *Peer) OnBlocks(msg *payload.BlockMessage) { - p.inch <- func() { - if p.config.OnBlock != nil { - p.config.OnBlock(p, msg) - } - } -} - -// OnVersion Listener will be called -// during the handshake, any error checking should be done here for the versionMessage. -// This should only ever be called during the handshake. Any other place and the peer will disconnect. -func (p *Peer) OnVersion(msg *payload.VersionMessage) error { - if msg.Nonce == p.config.Nonce { - p.conn.Close() - return errors.New("Self connection, disconnecting Peer") - } - p.versionKnown = true - p.port = msg.Port - p.services = msg.Services - p.userAgent = string(msg.UserAgent) - p.createdAt = time.Now() - p.relay = msg.Relay - return nil -} - -// OnHeaders is called when a Headers message is received -func (p *Peer) OnHeaders(msg *payload.HeadersMessage) { - fmt.Println("We have received the headers") - p.inch <- func() { - if p.config.OnHeader != nil { - p.config.OnHeader(p, msg) - } - } -} +// Outgoing Requests // RequestHeaders will write a getheaders to this peer func (p *Peer) RequestHeaders(hash util.Uint256) error { c := make(chan error, 0) p.outch <- func() { - p.Detector.AddMessage(command.GetHeaders) getHeaders, err := payload.NewGetHeadersMessage([]util.Uint256{hash}, util.Uint256{}) err = p.Write(getHeaders) + if err != nil { + p.Detector.AddMessage(command.GetHeaders) + } c <- err } return <-c @@ -437,17 +324,19 @@ func (p *Peer) RequestBlocks(hashes []util.Uint256) error { c := make(chan error, 0) p.outch <- func() { - p.Detector.AddMessage(command.GetData) getdata, err := payload.NewGetDataMessage(payload.InvTypeBlock) err = getdata.AddHashes(hashes) if err != nil { c <- err return } + err = p.Write(getdata) + if err != nil { + p.Detector.AddMessage(command.GetData) + } + c <- err } - return <-c - } diff --git a/pkg/peer/peer_test.go b/pkg/peer/peer_test.go index adcb4d342..fd27aeb55 100644 --- a/pkg/peer/peer_test.go +++ b/pkg/peer/peer_test.go @@ -1,7 +1,6 @@ package peer_test import ( - "fmt" "net" "testing" "time" @@ -21,11 +20,11 @@ func returnConfig() peer.LocalConfig { OnAddr := func(p *peer.Peer, msg *payload.AddrMessage) {} OnHeader := func(p *peer.Peer, msg *payload.HeadersMessage) {} - OnGetHeaders := func(msg *payload.GetHeadersMessage) {} + OnGetHeaders := func(p *peer.Peer, msg *payload.GetHeadersMessage) {} OnInv := func(p *peer.Peer, msg *payload.InvMessage) {} - OnGetData := func(msg *payload.GetDataMessage) {} + OnGetData := func(p *peer.Peer, msg *payload.GetDataMessage) {} OnBlock := func(p *peer.Peer, msg *payload.BlockMessage) {} - OnGetBlocks := func(msg *payload.GetBlocksMessage) {} + OnGetBlocks := func(p *peer.Peer, msg *payload.GetBlocksMessage) {} return peer.LocalConfig{ Net: protocol.MainNet, @@ -157,17 +156,9 @@ func TestConfigurations(t *testing.T) { assert.Equal(t, config.Services, p.Services()) - assert.Equal(t, config.UserAgent, p.UserAgent()) - assert.Equal(t, config.Relay, p.CanRelay()) assert.WithinDuration(t, time.Now(), p.CreatedAt(), 1*time.Second) - -} - -func TestHandshakeCancelled(t *testing.T) { - // These are the conditions which should invalidate the handshake. - // Make sure peer is disconnected. } func TestPeerDisconnect(t *testing.T) { @@ -178,21 +169,17 @@ func TestPeerDisconnect(t *testing.T) { inbound := true config := returnConfig() p := peer.NewPeer(conn, inbound, config) - fmt.Println("Calling disconnect") + p.Disconnect() - fmt.Println("Disconnect finished calling") - verack, _ := payload.NewVerackMessage() + verack, err := payload.NewVerackMessage() + assert.Nil(t, err) - fmt.Println(" We good here") + err = p.Write(verack) + assert.NotNil(t, err) - err := p.Write(verack) - - assert.NotEqual(t, err, nil) - - // Check if Stall detector is still running + // Check if stall detector is still running _, ok := <-p.Detector.Quitch assert.Equal(t, ok, false) - } func TestNotifyDisconnect(t *testing.T) { diff --git a/pkg/peer/responsehandlers.go b/pkg/peer/responsehandlers.go new file mode 100644 index 000000000..303ee0759 --- /dev/null +++ b/pkg/peer/responsehandlers.go @@ -0,0 +1,111 @@ +package peer + +import ( + "errors" + "time" + + "github.com/CityOfZion/neo-go/pkg/wire/payload" +) + +// OnGetData is called when a GetData message is received +func (p *Peer) OnGetData(msg *payload.GetDataMessage) { + p.inch <- func() { + if p.config.OnInv != nil { + p.config.OnGetData(p, msg) + } + } +} + +//OnTX is called when a TX message is received +func (p *Peer) OnTX(msg *payload.TXMessage) { + p.inch <- func() { + p.inch <- func() { + if p.config.OnTx != nil { + p.config.OnTx(p, msg) + } + } + } +} + +// OnInv is called when a Inv message is received +func (p *Peer) OnInv(msg *payload.InvMessage) { + p.inch <- func() { + if p.config.OnInv != nil { + p.config.OnInv(p, msg) + } + } +} + +// OnGetHeaders is called when a GetHeaders message is received +func (p *Peer) OnGetHeaders(msg *payload.GetHeadersMessage) { + p.inch <- func() { + if p.config.OnGetHeaders != nil { + p.config.OnGetHeaders(p, msg) + } + } +} + +// OnAddr is called when a Addr message is received +func (p *Peer) OnAddr(msg *payload.AddrMessage) { + p.inch <- func() { + if p.config.OnAddr != nil { + p.config.OnAddr(p, msg) + } + } +} + +// OnGetAddr is called when a GetAddr message is received +func (p *Peer) OnGetAddr(msg *payload.GetAddrMessage) { + p.inch <- func() { + if p.config.OnGetAddr != nil { + p.config.OnGetAddr(p, msg) + } + } +} + +// OnGetBlocks is called when a GetBlocks message is received +func (p *Peer) OnGetBlocks(msg *payload.GetBlocksMessage) { + p.inch <- func() { + if p.config.OnGetBlocks != nil { + p.config.OnGetBlocks(p, msg) + } + } +} + +// OnBlocks is called when a Blocks message is received +func (p *Peer) OnBlocks(msg *payload.BlockMessage) { + p.Detector.RemoveMessage(msg.Command()) + p.inch <- func() { + if p.config.OnBlock != nil { + p.config.OnBlock(p, msg) + } + } +} + +// OnHeaders is called when a Headers message is received +func (p *Peer) OnHeaders(msg *payload.HeadersMessage) { + p.Detector.RemoveMessage(msg.Command()) + p.inch <- func() { + if p.config.OnHeader != nil { + p.config.OnHeader(p, msg) + } + } +} + +// OnVersion Listener will be called +// during the handshake, any error checking should be done here for the versionMessage. +// This should only ever be called during the handshake. Any other place and the peer will disconnect. +func (p *Peer) OnVersion(msg *payload.VersionMessage) error { + if msg.Nonce == p.config.Nonce { + p.conn.Close() + return errors.New("self connection, disconnecting Peer") + } + p.versionKnown = true + p.port = msg.Port + p.services = msg.Services + p.userAgent = string(msg.UserAgent) + p.createdAt = time.Now() + p.relay = msg.Relay + p.startHeight = msg.StartHeight + return nil +} diff --git a/pkg/peer/stall/stall.go b/pkg/peer/stall/stall.go index fc19891fa..e69bcb442 100644 --- a/pkg/peer/stall/stall.go +++ b/pkg/peer/stall/stall.go @@ -61,6 +61,7 @@ func (d *Detector) loop() { d.lock.RUnlock() for _, deadline := range resp { if now.After(deadline) { + fmt.Println(resp) fmt.Println("Deadline passed") return } @@ -99,7 +100,7 @@ func (d *Detector) AddMessage(cmd command.Type) { // peer. This will remove the pendingresponse message from the map. // The command passed through is the command we received func (d *Detector) RemoveMessage(cmd command.Type) { - cmds := d.addMessage(cmd) + cmds := d.removeMessage(cmd) d.lock.Lock() for _, cmd := range cmds { delete(d.responses, cmd) @@ -137,10 +138,8 @@ func (d *Detector) addMessage(cmd command.Type) []command.Type { case command.GetAddr: // We now will expect a Headers Message cmds = append(cmds, command.Addr) - case command.GetData: // We will now expect a block/tx message - // We can optimise this by including the exact inventory type, however it is not needed cmds = append(cmds, command.Block) cmds = append(cmds, command.TX) case command.GetBlocks: @@ -159,19 +158,18 @@ func (d *Detector) removeMessage(cmd command.Type) []command.Type { switch cmd { case command.Block: - // We will now expect a block/tx message + // We will now remove a block and tx message cmds = append(cmds, command.Block) cmds = append(cmds, command.TX) case command.TX: - // We will now expect a block/tx message + // We will now remove a block and tx message cmds = append(cmds, command.Block) cmds = append(cmds, command.TX) - case command.GetBlocks: - // we will now expect a inv message - cmds = append(cmds, command.Inv) - default: + case command.Verack: // We will now expect a verack cmds = append(cmds, cmd) + default: + cmds = append(cmds, cmd) } return cmds } diff --git a/pkg/peer/stall/stall_test.go b/pkg/peer/stall/stall_test.go index 4d5494e12..83de4e3a0 100644 --- a/pkg/peer/stall/stall_test.go +++ b/pkg/peer/stall/stall_test.go @@ -22,7 +22,7 @@ func TestAddRemoveMessage(t *testing.T) { assert.Equal(t, 1, len(mp)) assert.IsType(t, time.Time{}, mp[command.GetAddr]) - d.RemoveMessage(command.GetAddr) + d.RemoveMessage(command.Addr) mp = d.GetMessages() assert.Equal(t, 0, len(mp)) From 8afec1ea45cf18c6e5237271ea3c70b285b1722d Mon Sep 17 00:00:00 2001 From: decentralisedkev <37423678+decentralisedkev@users.noreply.github.com> Date: Thu, 28 Mar 2019 19:46:31 +0000 Subject: [PATCH 11/24] [Peer] Add peer manager (#241) * [PeerMgr] - Add basic peer manager --- pkg/peer/peer.go | 3 +- pkg/peermgr/peermgr.go | 122 ++++++++++++++++++++++++ pkg/peermgr/peermgr_test.go | 179 ++++++++++++++++++++++++++++++++++++ 3 files changed, 302 insertions(+), 2 deletions(-) create mode 100644 pkg/peermgr/peermgr.go create mode 100644 pkg/peermgr/peermgr_test.go diff --git a/pkg/peer/peer.go b/pkg/peer/peer.go index 94f9c685c..d1594d6de 100644 --- a/pkg/peer/peer.go +++ b/pkg/peer/peer.go @@ -175,11 +175,10 @@ func (p *Peer) IsVerackReceived() bool { //NotifyDisconnect returns once the peer has disconnected // Blocking -func (p *Peer) NotifyDisconnect() bool { +func (p *Peer) NotifyDisconnect() { fmt.Println("Peer has not disconnected yet") <-p.quitch fmt.Println("Peer has just disconnected") - return true } //End of Exposed API functions// diff --git a/pkg/peermgr/peermgr.go b/pkg/peermgr/peermgr.go new file mode 100644 index 000000000..047a109c3 --- /dev/null +++ b/pkg/peermgr/peermgr.go @@ -0,0 +1,122 @@ +package peermgr + +import ( + "errors" + "sync" + + "github.com/CityOfZion/neo-go/pkg/wire/command" + + "github.com/CityOfZion/neo-go/pkg/wire/util" +) + +var ( + //ErrNoAvailablePeers is returned when a request for data from a peer is invoked + // but there are no available peers to request data from + ErrNoAvailablePeers = errors.New("there are no available peers to interact with") + + // ErrUnknownPeer is returned when a peer that the peer manager does not know about + // sends a message to this node + ErrUnknownPeer = errors.New("this peer has not been registered with the peer manager") +) + +//mPeer represents a peer that is managed by the peer manager +type mPeer interface { + Disconnect() + RequestBlocks([]util.Uint256) error + RequestHeaders(util.Uint256) error + NotifyDisconnect() +} + +type peerstats struct { + requests map[command.Type]bool +} + +//PeerMgr manages all peers that the node is connected to +type PeerMgr struct { + pLock sync.RWMutex + peers map[mPeer]peerstats +} + +//New returns a new peermgr object +func New() *PeerMgr { + return &PeerMgr{ + peers: make(map[mPeer]peerstats), + } +} + +// AddPeer adds a peer to the list of managed peers +func (pmgr *PeerMgr) AddPeer(peer mPeer) { + + pmgr.pLock.Lock() + defer pmgr.pLock.Unlock() + if _, exists := pmgr.peers[peer]; exists { + return + } + pmgr.peers[peer] = peerstats{requests: make(map[command.Type]bool)} + go pmgr.onDisconnect(peer) +} + +//MsgReceived notifies the peer manager that we have received a +// message from a peer +func (pmgr *PeerMgr) MsgReceived(peer mPeer, cmd command.Type) error { + pmgr.pLock.Lock() + defer pmgr.pLock.Unlock() + val, ok := pmgr.peers[peer] + if !ok { + + go func() { + peer.NotifyDisconnect() + }() + + peer.Disconnect() + return ErrUnknownPeer + } + val.requests[cmd] = false + + return nil +} + +// Len returns the amount of peers that the peer manager +//currently knows about +func (pmgr *PeerMgr) Len() int { + pmgr.pLock.Lock() + defer pmgr.pLock.Unlock() + return len(pmgr.peers) +} + +// RequestBlock will request a block from the most +// available peer. Then update it's stats, so we know that +// this peer is busy +func (pmgr *PeerMgr) RequestBlock(hash util.Uint256) error { + return pmgr.callPeerForCmd(command.Block, func(p mPeer) error { + return p.RequestBlocks([]util.Uint256{hash}) + }) +} + +// RequestHeaders will request a headers from the most available peer. +func (pmgr *PeerMgr) RequestHeaders(hash util.Uint256) error { + return pmgr.callPeerForCmd(command.Headers, func(p mPeer) error { + return p.RequestHeaders(hash) + }) +} + +func (pmgr *PeerMgr) callPeerForCmd(cmd command.Type, f func(p mPeer) error) error { + pmgr.pLock.Lock() + defer pmgr.pLock.Unlock() + for peer, stats := range pmgr.peers { + if !stats.requests[cmd] { + stats.requests[cmd] = true + return f(peer) + } + } + return ErrNoAvailablePeers +} +func (pmgr *PeerMgr) onDisconnect(p mPeer) { + + // Blocking until peer is disconnected + p.NotifyDisconnect() + + pmgr.pLock.Lock() + delete(pmgr.peers, p) + pmgr.pLock.Unlock() +} diff --git a/pkg/peermgr/peermgr_test.go b/pkg/peermgr/peermgr_test.go new file mode 100644 index 000000000..86d5be41c --- /dev/null +++ b/pkg/peermgr/peermgr_test.go @@ -0,0 +1,179 @@ +package peermgr + +import ( + "testing" + + "github.com/CityOfZion/neo-go/pkg/wire/command" + + "github.com/CityOfZion/neo-go/pkg/wire/util" + "github.com/stretchr/testify/assert" +) + +type peer struct { + quit chan bool + nonce int + disconnected bool + blockRequested int + headersRequested int +} + +func (p *peer) Disconnect() { + p.disconnected = true + p.quit <- true +} +func (p *peer) RequestBlocks([]util.Uint256) error { + p.blockRequested++ + return nil +} +func (p *peer) RequestHeaders(util.Uint256) error { + p.headersRequested++ + return nil +} +func (p *peer) NotifyDisconnect() { + <-p.quit +} + +func TestAddPeer(t *testing.T) { + pmgr := New() + + peerA := &peer{nonce: 1} + peerB := &peer{nonce: 2} + peerC := &peer{nonce: 3} + + pmgr.AddPeer(peerA) + pmgr.AddPeer(peerB) + pmgr.AddPeer(peerC) + pmgr.AddPeer(peerC) + + assert.Equal(t, 3, pmgr.Len()) +} + +func TestRequestBlocks(t *testing.T) { + pmgr := New() + + peerA := &peer{nonce: 1} + peerB := &peer{nonce: 2} + peerC := &peer{nonce: 3} + + pmgr.AddPeer(peerA) + pmgr.AddPeer(peerB) + pmgr.AddPeer(peerC) + + err := pmgr.RequestBlock(util.Uint256{}) + assert.Nil(t, err) + + err = pmgr.RequestBlock(util.Uint256{}) + assert.Nil(t, err) + + err = pmgr.RequestBlock(util.Uint256{}) + assert.Nil(t, err) + + // Since the peer manager did not get a MsgReceived + // in between the block requests + // a request should be sent to all peers + + assert.Equal(t, 1, peerA.blockRequested) + assert.Equal(t, 1, peerB.blockRequested) + assert.Equal(t, 1, peerC.blockRequested) + + // Since the peer manager still has not received a MsgReceived + // another call to request blocks, will return a NoAvailablePeerError + + err = pmgr.RequestBlock(util.Uint256{}) + assert.Equal(t, ErrNoAvailablePeers, err) + + // If we tell the peer manager that peerA has given us a block + // then send another BlockRequest. It will go to peerA + // since the other two peers are still busy with their + // block requests + + pmgr.MsgReceived(peerA, command.Block) + err = pmgr.RequestBlock(util.Uint256{}) + assert.Nil(t, err) + + assert.Equal(t, 2, peerA.blockRequested) + assert.Equal(t, 1, peerB.blockRequested) + assert.Equal(t, 1, peerC.blockRequested) +} +func TestRequestHeaders(t *testing.T) { + pmgr := New() + + peerA := &peer{nonce: 1} + peerB := &peer{nonce: 2} + peerC := &peer{nonce: 3} + + pmgr.AddPeer(peerA) + pmgr.AddPeer(peerB) + pmgr.AddPeer(peerC) + + err := pmgr.RequestHeaders(util.Uint256{}) + assert.Nil(t, err) + + err = pmgr.RequestHeaders(util.Uint256{}) + assert.Nil(t, err) + + err = pmgr.RequestHeaders(util.Uint256{}) + assert.Nil(t, err) + + // Since the peer manager did not get a MsgReceived + // in between the header requests + // a request should be sent to all peers + + assert.Equal(t, 1, peerA.headersRequested) + assert.Equal(t, 1, peerB.headersRequested) + assert.Equal(t, 1, peerC.headersRequested) + + // Since the peer manager still has not received a MsgReceived + // another call to request header, will return a NoAvailablePeerError + + err = pmgr.RequestHeaders(util.Uint256{}) + assert.Equal(t, ErrNoAvailablePeers, err) + + // If we tell the peer manager that peerA has given us a block + // then send another BlockRequest. It will go to peerA + // since the other two peers are still busy with their + // block requests + + err = pmgr.MsgReceived(peerA, command.Headers) + assert.Nil(t, err) + err = pmgr.RequestHeaders(util.Uint256{}) + assert.Nil(t, err) + + assert.Equal(t, 2, peerA.headersRequested) + assert.Equal(t, 1, peerB.headersRequested) + assert.Equal(t, 1, peerC.headersRequested) +} + +func TestUnknownPeer(t *testing.T) { + pmgr := New() + + unknownPeer := &peer{ + disconnected: false, + quit: make(chan bool), + } + + err := pmgr.MsgReceived(unknownPeer, command.Block) + assert.Equal(t, true, unknownPeer.disconnected) + assert.Equal(t, ErrUnknownPeer, err) +} + +func TestNotifyDisconnect(t *testing.T) { + pmgr := New() + + peerA := &peer{ + nonce: 1, + quit: make(chan bool), + } + + pmgr.AddPeer(peerA) + + if pmgr.Len() != 1 { + t.Fail() + } + + peerA.Disconnect() + + if pmgr.Len() != 0 { + t.Fail() + } +} From 493d8f3d951cdefee2a970a5dd302f6d5b64597b Mon Sep 17 00:00:00 2001 From: decentralisedkev <37423678+decentralisedkev@users.noreply.github.com> Date: Thu, 28 Mar 2019 20:23:50 +0000 Subject: [PATCH 12/24] [chain] Refactor, add chaincfg and database initialisation (#243) * [chain] - Add basic chain cfg parameters - Added logic to insert genesis block, if it is a fresh database - changed SaveBlock to ProcessBlock - changed SaveHeaders to ProcessHeaders - Changed parameter from a wire message to the payload, for header and block processing - Added check in chain for when the block is in the future, i.e. not at the tip of the chain - Added custom error returns, to distinguish between a database error and a validation error --- pkg/chain/chain.go | 107 +++++++++++++++++++++++----------- pkg/chain/errors.go | 19 ++++++ pkg/chaincfg/chaincfg.go | 44 ++++++++++++++ pkg/chaincfg/chaincfg_test.go | 13 +++++ 4 files changed, 149 insertions(+), 34 deletions(-) create mode 100644 pkg/chain/errors.go create mode 100644 pkg/chaincfg/chaincfg.go create mode 100644 pkg/chaincfg/chaincfg_test.go diff --git a/pkg/chain/chain.go b/pkg/chain/chain.go index c1492cb8f..c44764388 100644 --- a/pkg/chain/chain.go +++ b/pkg/chain/chain.go @@ -1,9 +1,13 @@ package chain import ( - "errors" + "fmt" + "github.com/pkg/errors" + + "github.com/CityOfZion/neo-go/pkg/chaincfg" "github.com/CityOfZion/neo-go/pkg/wire/payload/transaction" + "github.com/CityOfZion/neo-go/pkg/wire/protocol" "github.com/CityOfZion/neo-go/pkg/database" "github.com/CityOfZion/neo-go/pkg/wire/payload" @@ -12,58 +16,89 @@ import ( var ( // ErrBlockAlreadyExists happens when you try to save the same block twice ErrBlockAlreadyExists = errors.New("this block has already been saved in the database") + + // ErrFutureBlock happens when you try to save a block that is not the next block sequentially + ErrFutureBlock = errors.New("this is not the next block sequentially, that should be added to the chain") ) // Chain represents a blockchain instance type Chain struct { - db *Chaindb + Db *Chaindb } -//New returns a new chain instance -func New(db database.Database) *Chain { - return &Chain{ - db: &Chaindb{db}, +// New returns a new chain instance +func New(db database.Database, magic protocol.Magic) (*Chain, error) { + + chain := &Chain{ + Db: &Chaindb{db}, } + + // Get last header saved to see if this is a fresh database + _, err := chain.Db.GetLastHeader() + if err == nil { + return chain, nil + } + + if err != database.ErrNotFound { + return nil, err + } + + // We have a database.ErrNotFound. Insert the genesisBlock + fmt.Printf("Starting a fresh database for %s\n", magic.String()) + + params, err := chaincfg.NetParams(magic) + if err != nil { + return nil, err + } + err = chain.Db.saveHeader(¶ms.GenesisBlock.BlockBase) + if err != nil { + return nil, err + } + err = chain.Db.saveBlock(params.GenesisBlock, true) + if err != nil { + return nil, err + } + return chain, nil } -// SaveBlock verifies and saves the block in the database +// ProcessBlock verifies and saves the block in the database // XXX: for now we will just save without verifying the block // This function is called by the server and if an error is returned then // the server informs the sync manager to redownload the block // XXX:We should also check if the header is already saved in the database // If not, then we need to validate the header with the rest of the chain // For now we re-save the header -func (c *Chain) SaveBlock(msg payload.BlockMessage) error { - err := c.VerifyBlock(msg.Block) - if err != nil { - return err - } - //XXX(Issue): We can either check the hash here for genesisblock. - //We most likely will have it anyways after validation/ We can return it from VerifyBlock - // Or we can do it somewhere in startup, performance benefits - // won't be that big since it's just a bytes.Equal. - // so it's more about which is more readable and where it makes sense to put - return c.db.saveBlock(msg.Block, false) -} +func (c *Chain) ProcessBlock(block payload.Block) error { -// VerifyBlock verifies whether a block is valid according -// to the rules of consensus -func (c *Chain) VerifyBlock(block payload.Block) error { - - // Check if we already have this block - // XXX: We can optimise by implementing a Has method + // Check if we already have this block saved + // XXX: We can optimise by implementing a Has() method // caching the last block in memory - lastBlock, err := c.db.GetLastBlock() + lastBlock, err := c.Db.GetLastBlock() if err != nil { return err } - // Check if we have already saved this block - // by looking if the latest block height is more than - // incoming block height if lastBlock.Index > block.Index { return ErrBlockAlreadyExists } + if block.Index > lastBlock.Index+1 { + return ErrFutureBlock + } + + err = c.verifyBlock(block) + if err != nil { + return ValidationError{err.Error()} + } + err = c.Db.saveBlock(block, false) + if err != nil { + return DatabaseError{err.Error()} + } + return nil +} + +// VerifyBlock verifies whether a block is valid according +// to the rules of consensus +func (c *Chain) verifyBlock(block payload.Block) error { return nil } @@ -73,14 +108,18 @@ func (c *Chain) VerifyTx(tx transaction.Transactioner) error { return nil } -// SaveHeaders will save the set of headers without validating -func (c *Chain) SaveHeaders(msg payload.HeadersMessage) error { +// ProcessHeaders will save the set of headers without validating +func (c *Chain) ProcessHeaders(hdrs []*payload.BlockBase) error { - err := c.verifyHeaders(msg.Headers) + err := c.verifyHeaders(hdrs) if err != nil { - return err + return ValidationError{err.Error()} } - return c.db.saveHeaders(msg.Headers) + err = c.Db.saveHeaders(hdrs) + if err != nil { + return DatabaseError{err.Error()} + } + return nil } // verifyHeaders will be used to verify a batch of headers diff --git a/pkg/chain/errors.go b/pkg/chain/errors.go new file mode 100644 index 000000000..627a304ba --- /dev/null +++ b/pkg/chain/errors.go @@ -0,0 +1,19 @@ +package chain + +// ValidationError occurs when verificatio of the object fails +type ValidationError struct { + msg string +} + +func (v ValidationError) Error() string { + return v.msg +} + +// DatabaseError occurs when the chain fails to save the object in the database +type DatabaseError struct { + msg string +} + +func (d DatabaseError) Error() string { + return d.msg +} diff --git a/pkg/chaincfg/chaincfg.go b/pkg/chaincfg/chaincfg.go new file mode 100644 index 000000000..ff6024c6f --- /dev/null +++ b/pkg/chaincfg/chaincfg.go @@ -0,0 +1,44 @@ +package chaincfg + +import ( + "bytes" + "encoding/hex" + + "github.com/CityOfZion/neo-go/pkg/wire/payload" + "github.com/CityOfZion/neo-go/pkg/wire/protocol" +) + +// Params are the parameters needed to setup the network +type Params struct { + GenesisBlock payload.Block +} + +//NetParams returns the parameters for the chosen network magic +func NetParams(magic protocol.Magic) (Params, error) { + switch magic { + case protocol.MainNet: + return mainnet() + default: + return mainnet() + } +} + +//Mainnet returns the parameters needed for mainnet +func mainnet() (Params, error) { + rawHex := "000000000000000000000000000000000000000000000000000000000000000000000000f41bc036e39b0d6b0579c851c6fde83af802fa4e57bec0bc3365eae3abf43f8065fc8857000000001dac2b7c0000000059e75d652b5d3827bf04c165bbe9ef95cca4bf55010001510400001dac2b7c00000000400000455b7b226c616e67223a227a682d434e222c226e616d65223a22e5b08fe89a81e882a1227d2c7b226c616e67223a22656e222c226e616d65223a22416e745368617265227d5d0000c16ff28623000000da1745e9b549bd0bfa1a569971c77eba30cd5a4b00000000400001445b7b226c616e67223a227a682d434e222c226e616d65223a22e5b08fe89a81e5b881227d2c7b226c616e67223a22656e222c226e616d65223a22416e74436f696e227d5d0000c16ff286230008009f7fd096d37ed2c0e3f7f0cfc924beef4ffceb680000000001000000019b7cffdaa674beae0f930ebe6085af9093e5fe56b34a5c220ccdcf6efc336fc50000c16ff28623005fa99d93303775fe50ca119c327759313eccfa1c01000151" + rawBytes, err := hex.DecodeString(rawHex) + if err != nil { + return Params{}, err + } + reader := bytes.NewReader(rawBytes) + + block := payload.Block{} + err = block.Decode(reader) + if err != nil { + return Params{}, err + } + + return Params{ + GenesisBlock: block, + }, nil +} diff --git a/pkg/chaincfg/chaincfg_test.go b/pkg/chaincfg/chaincfg_test.go new file mode 100644 index 000000000..043350b81 --- /dev/null +++ b/pkg/chaincfg/chaincfg_test.go @@ -0,0 +1,13 @@ +package chaincfg + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestMainnet(t *testing.T) { + p, err := mainnet() + assert.Nil(t, err) + assert.Equal(t, p.GenesisBlock.Hash.ReverseString(), "d42561e3d30e15be6400b6df2f328e02d2bf6354c41dce433bc57687c82144bf") +} From cb21c66316d8da57a5d0d2485b620a00e3548188 Mon Sep 17 00:00:00 2001 From: decentralisedkev <37423678+decentralisedkev@users.noreply.github.com> Date: Thu, 28 Mar 2019 21:22:17 +0000 Subject: [PATCH 13/24] Syncmgr: Implement synchronisation manager (#249) * [syncmgr] - Add blockmode, normal mode, headermode - Add config file - Add test files - removed RequestBlocks and RequestHeaders from peers, as we will use the peermanager for this - OnHeaders and OnBlock in syncmgr, now return errors - refactored all tests to use a convenience method to return a syncmgr and testHelper --- pkg/syncmgr/blockmode.go | 55 +++++++++++ pkg/syncmgr/config.go | 47 +++++++++ pkg/syncmgr/headermode.go | 41 ++++++++ pkg/syncmgr/mockhelpers_test.go | 113 +++++++++++++++++++++ pkg/syncmgr/normalmode.go | 59 +++++++++++ pkg/syncmgr/syncmgr.go | 135 ++++++++++++++++++++++++++ pkg/syncmgr/syncmgr_onblock_test.go | 97 ++++++++++++++++++ pkg/syncmgr/syncmgr_onheaders_test.go | 117 ++++++++++++++++++++++ 8 files changed, 664 insertions(+) create mode 100644 pkg/syncmgr/blockmode.go create mode 100644 pkg/syncmgr/config.go create mode 100644 pkg/syncmgr/headermode.go create mode 100644 pkg/syncmgr/mockhelpers_test.go create mode 100644 pkg/syncmgr/normalmode.go create mode 100644 pkg/syncmgr/syncmgr.go create mode 100644 pkg/syncmgr/syncmgr_onblock_test.go create mode 100644 pkg/syncmgr/syncmgr_onheaders_test.go diff --git a/pkg/syncmgr/blockmode.go b/pkg/syncmgr/blockmode.go new file mode 100644 index 000000000..2e5dadd6f --- /dev/null +++ b/pkg/syncmgr/blockmode.go @@ -0,0 +1,55 @@ +package syncmgr + +import ( + "github.com/CityOfZion/neo-go/pkg/chain" + "github.com/CityOfZion/neo-go/pkg/wire/payload" +) + +// blockModeOnBlock is called when the sync manager is block mode +// and receives a block. +func (s *Syncmgr) blockModeOnBlock(peer SyncPeer, block payload.Block) error { + + // Process Block + err := s.cfg.ProcessBlock(block) + + if err == chain.ErrFutureBlock { + // XXX(Optimisation): We can cache future blocks in blockmode, if we have the corresponding header + // We can have the server cache them and sort out the semantics for when to send them to the chain + // Server can listen on chain for when a new block is saved + // or we could embed a struct in this syncmgr called blockCache, syncmgr can just tell it when it has processed + //a block and we can call ProcessBlock + return err + } + + if err != nil && err != chain.ErrBlockAlreadyExists { + return s.cfg.FetchBlockAgain(block.Hash) + } + + // Check if blockhashReceived == the header hash from last get headers this node performed + // if not then increment and request next block + if s.headerHash != block.Hash { + nextHash, err := s.cfg.GetNextBlockHash() + if err != nil { + return err + } + err = s.cfg.RequestBlock(nextHash) + return err + } + + // If we are caught up then go into normal mode + diff := peer.Height() - block.Index + if diff <= cruiseHeight { + s.syncmode = normalMode + s.timer.Reset(blockTimer) + return nil + } + + // If not then we go back into headersMode and request more headers. + s.syncmode = headersMode + return s.cfg.RequestHeaders(block.Hash) +} + +func (s *Syncmgr) blockModeOnHeaders(peer SyncPeer, hdrs []*payload.BlockBase) error { + // We ignore headers when in this mode + return nil +} diff --git a/pkg/syncmgr/config.go b/pkg/syncmgr/config.go new file mode 100644 index 000000000..bc921b33b --- /dev/null +++ b/pkg/syncmgr/config.go @@ -0,0 +1,47 @@ +package syncmgr + +import ( + "github.com/CityOfZion/neo-go/pkg/wire/payload" + "github.com/CityOfZion/neo-go/pkg/wire/util" +) + +// Config is the configuration file for the sync manager +type Config struct { + + // Chain functions + ProcessBlock func(msg payload.Block) error + ProcessHeaders func(hdrs []*payload.BlockBase) error + + // RequestHeaders will send a getHeaders request + // with the hash passed in as a parameter + RequestHeaders func(hash util.Uint256) error + + //RequestBlock will send a getdata request for the block + // with the hash passed as a parameter + RequestBlock func(hash util.Uint256) error + + // GetNextBlockHash returns the block hash of the header infront of thr block + // at the tip of this nodes chain. This assumes that the node is not in sync + GetNextBlockHash func() (util.Uint256, error) + + // GetBestBlockHash gets the block hash of the last saved block. + GetBestBlockHash func() (util.Uint256, error) + + // AskForNewBlocks will send out a message to the network + // asking for new blocks + AskForNewBlocks func() + + // FetchHeadersAgain is called when a peer has provided headers that have not + // validated properly. We pass in the hash of the first header + FetchHeadersAgain func(util.Uint256) error + + // FetchHeadersAgain is called when a peer has provided a block that has not + // validated properly. We pass in the hash of the block + FetchBlockAgain func(util.Uint256) error +} + +// SyncPeer represents a peer on the network +// that this node can sync with +type SyncPeer interface { + Height() uint32 +} diff --git a/pkg/syncmgr/headermode.go b/pkg/syncmgr/headermode.go new file mode 100644 index 000000000..898dc933d --- /dev/null +++ b/pkg/syncmgr/headermode.go @@ -0,0 +1,41 @@ +package syncmgr + +import ( + "github.com/CityOfZion/neo-go/pkg/chain" + "github.com/CityOfZion/neo-go/pkg/wire/payload" +) + +// headersModeOnHeaders is called when the sync manager is headers mode +// and receives a header. +func (s *Syncmgr) headersModeOnHeaders(peer SyncPeer, hdrs []*payload.BlockBase) error { + // If we are in Headers mode, then we just need to process the headers + // Note: For the un-optimised version, we move straight to blocksOnly mode + + firstHash := hdrs[0].Hash + + err := s.cfg.ProcessHeaders(hdrs) + if err == nil { + // Update syncmgr last header + s.headerHash = hdrs[len(hdrs)-1].Hash + + s.syncmode = blockMode + return s.cfg.RequestBlock(firstHash) + } + + // Check whether it is a validation error, or a database error + if _, ok := err.(*chain.ValidationError); ok { + // If we get a validation error we re-request the headers + // the method will automatically fetch from a different peer + // XXX: Add increment banScore for this peer + return s.cfg.FetchHeadersAgain(firstHash) + } + // This means it is a database error. We have no way to recover from this. + panic(err.Error()) +} + +// headersModeOnBlock is called when the sync manager is headers mode +// and receives a block. +func (s *Syncmgr) headersModeOnBlock(peer SyncPeer, block payload.Block) error { + // While in headers mode, ignore any blocks received + return nil +} diff --git a/pkg/syncmgr/mockhelpers_test.go b/pkg/syncmgr/mockhelpers_test.go new file mode 100644 index 000000000..8c84225ac --- /dev/null +++ b/pkg/syncmgr/mockhelpers_test.go @@ -0,0 +1,113 @@ +package syncmgr + +import ( + "crypto/rand" + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/CityOfZion/neo-go/pkg/wire/payload" + "github.com/CityOfZion/neo-go/pkg/wire/util" +) + +type syncTestHelper struct { + blocksProcessed int + headersProcessed int + newBlockRequest int + headersFetchRequest int + blockFetchRequest int + err error +} + +func (s *syncTestHelper) ProcessBlock(msg payload.Block) error { + s.blocksProcessed++ + return s.err +} +func (s *syncTestHelper) ProcessHeaders(hdrs []*payload.BlockBase) error { + s.headersProcessed = s.headersProcessed + len(hdrs) + return s.err +} + +func (s *syncTestHelper) GetNextBlockHash() (util.Uint256, error) { + return util.Uint256{}, s.err +} + +func (s *syncTestHelper) AskForNewBlocks() { + s.newBlockRequest++ +} + +func (s *syncTestHelper) FetchHeadersAgain(util.Uint256) error { + s.headersFetchRequest++ + return s.err +} + +func (s *syncTestHelper) FetchBlockAgain(util.Uint256) error { + s.blockFetchRequest++ + return s.err +} + +func (s *syncTestHelper) RequestBlock(util.Uint256) error { + s.blockFetchRequest++ + return s.err +} + +func (s *syncTestHelper) RequestHeaders(util.Uint256) error { + s.headersFetchRequest++ + return s.err +} + +type mockPeer struct { + height uint32 +} + +func (p *mockPeer) Height() uint32 { return p.height } + +func randomHeadersMessage(t *testing.T, num int) *payload.HeadersMessage { + var hdrs []*payload.BlockBase + + for i := 0; i < num; i++ { + hash := randomUint256(t) + hdr := &payload.BlockBase{Hash: hash} + hdrs = append(hdrs, hdr) + } + + hdrsMsg, err := payload.NewHeadersMessage() + assert.Nil(t, err) + + hdrsMsg.Headers = hdrs + + return hdrsMsg +} + +func randomUint256(t *testing.T) util.Uint256 { + hash := make([]byte, 32) + _, err := rand.Read(hash) + assert.Nil(t, err) + + u, err := util.Uint256DecodeBytes(hash) + assert.Nil(t, err) + return u +} + +func setupSyncMgr(mode mode) (*Syncmgr, *syncTestHelper) { + helper := &syncTestHelper{} + + cfg := &Config{ + ProcessBlock: helper.ProcessBlock, + ProcessHeaders: helper.ProcessHeaders, + + GetNextBlockHash: helper.GetNextBlockHash, + AskForNewBlocks: helper.AskForNewBlocks, + + FetchHeadersAgain: helper.FetchHeadersAgain, + FetchBlockAgain: helper.FetchBlockAgain, + + RequestBlock: helper.RequestBlock, + RequestHeaders: helper.RequestHeaders, + } + + syncmgr := New(cfg) + syncmgr.syncmode = mode + + return syncmgr, helper +} diff --git a/pkg/syncmgr/normalmode.go b/pkg/syncmgr/normalmode.go new file mode 100644 index 000000000..218d6151b --- /dev/null +++ b/pkg/syncmgr/normalmode.go @@ -0,0 +1,59 @@ +package syncmgr + +import ( + "github.com/CityOfZion/neo-go/pkg/wire/payload" +) + +func (s *Syncmgr) normalModeOnHeaders(peer SyncPeer, hdrs []*payload.BlockBase) error { + // If in normal mode, first process the headers + err := s.cfg.ProcessHeaders(hdrs) + if err != nil { + // If something went wrong with processing the headers + // Ask another peer for the headers. + //XXX: Increment banscore for this peer + return s.cfg.FetchHeadersAgain(hdrs[0].Hash) + } + + lenHeaders := len(hdrs) + firstHash := hdrs[0].Hash + lastHash := hdrs[lenHeaders-1].Hash + + // Update syncmgr latest header + s.headerHash = lastHash + + // If there are 2k headers, then ask for more headers and switch back to headers mode. + if lenHeaders == 2000 { + s.syncmode = headersMode + return s.cfg.RequestHeaders(lastHash) + } + + // Ask for the corresponding block iff there is < 2k headers + // then switch to blocksMode + // Bounds state that len > 1 && len!= 2000 & maxHeadersInMessage == 2000 + // This means that we have less than 2k headers + s.syncmode = blockMode + return s.cfg.RequestBlock(firstHash) +} + +// normalModeOnBlock is called when the sync manager is normal mode +// and receives a block. +func (s *Syncmgr) normalModeOnBlock(peer SyncPeer, block payload.Block) error { + // stop the timer that periodically asks for blocks + s.timer.Stop() + + // process block + err := s.cfg.ProcessBlock(block) + if err != nil { + s.timer.Reset(blockTimer) + return s.cfg.FetchBlockAgain(block.Hash) + } + + diff := peer.Height() - block.Index + if diff > trailingHeight { + s.syncmode = headersMode + return s.cfg.RequestHeaders(block.Hash) + } + + s.timer.Reset(blockTimer) + return nil +} diff --git a/pkg/syncmgr/syncmgr.go b/pkg/syncmgr/syncmgr.go new file mode 100644 index 000000000..bb06dfa7e --- /dev/null +++ b/pkg/syncmgr/syncmgr.go @@ -0,0 +1,135 @@ +package syncmgr + +import ( + "fmt" + "time" + + "github.com/CityOfZion/neo-go/pkg/wire/payload" + "github.com/CityOfZion/neo-go/pkg/wire/util" +) + +type mode uint8 + +// Note: this is the unoptimised version without parallel sync +// The algorithm for the unoptimsied version is simple: +// Download 2000 headers, then download the blocks for those headers +// Once those blocks are downloaded, we repeat the process again +// Until we are nomore than one block behind the tip. +// Once this happens, we switch into normal mode. +//In normal mode, we have a timer on for X seconds and ask nodes for blocks and also to doublecheck +// if we are behind once the timer runs out. +// The timer restarts whenever we receive a block. +// The parameter X should be approximately the time it takes the network to reach consensus + +//blockTimer approximates to how long it takes to reach consensus and propagate +// a block in the network. Once a node has synchronised with the network, he will +// ask the network for a newblock every blockTimer +const blockTimer = 20 * time.Second + +// trailingHeight indicates how many blocks the node has to be behind by +// before he switches to headersMode. +const trailingHeight = 100 + +// indicates how many blocks the node has to be behind by +// before he switches to normalMode and fetches blocks every X seconds. +const cruiseHeight = 0 + +const ( + headersMode mode = 1 + blockMode mode = 2 + normalMode mode = 3 +) + +//Syncmgr keeps the node in sync with the rest of the network +type Syncmgr struct { + syncmode mode + cfg *Config + timer *time.Timer + + // headerHash is the hash of the last header in the last OnHeaders message that we received. + // When receiving blocks, we can use this to determine whether the node has downloaded + // all of the blocks for the last headers messages + headerHash util.Uint256 +} + +// New creates a new sync manager +func New(cfg *Config) *Syncmgr { + + newBlockTimer := time.AfterFunc(blockTimer, func() { + cfg.AskForNewBlocks() + }) + newBlockTimer.Stop() + + return &Syncmgr{ + syncmode: headersMode, + cfg: cfg, + timer: newBlockTimer, + } +} + +// OnHeader is called when the node receives a headers message +func (s *Syncmgr) OnHeader(peer SyncPeer, msg *payload.HeadersMessage) error { + + // XXX(Optimisation): First check if we actually need these headers + // Check the last header in msg and then check what our latest header that was saved is + // If our latest header is above the lastHeader, then we do not save it + // We could also have that our latest header is above only some of the headers. + // In this case, we should remove the headers that we already have + + if len(msg.Headers) == 0 { + // XXX: Increment banScore for this peer, for sending empty headers message + return nil + } + + var err error + + switch s.syncmode { + case headersMode: + err = s.headersModeOnHeaders(peer, msg.Headers) + case blockMode: + err = s.blockModeOnHeaders(peer, msg.Headers) + case normalMode: + err = s.normalModeOnHeaders(peer, msg.Headers) + default: + err = s.headersModeOnHeaders(peer, msg.Headers) + } + + // XXX(Kev):The only meaningful error here would be if the peer + // we re-requested blocks from failed. In the next iteration, this will be handled + // by the peer manager, who will only return an error, if we are connected to no peers. + // Upon re-alising this, the node will then send out GetAddresses to the network and + // syncing will be resumed, once we find peers to connect to. + + hdr := msg.Headers[len(msg.Headers)-1] + fmt.Printf("Finished processing headers. LastHash in set was: %s\n ", hdr.Hash.ReverseString()) + + return err +} + +// OnBlock is called when the node receives a block +func (s *Syncmgr) OnBlock(peer SyncPeer, msg *payload.BlockMessage) error { + fmt.Printf("Block received with height %d\n", msg.Block.Index) + + var err error + + switch s.syncmode { + case headersMode: + err = s.headersModeOnBlock(peer, msg.Block) + case blockMode: + err = s.blockModeOnBlock(peer, msg.Block) + case normalMode: + err = s.normalModeOnBlock(peer, msg.Block) + default: + err = s.headersModeOnBlock(peer, msg.Block) + } + + fmt.Printf("Processed Block with height %d\n", msg.Block.Index) + + return err +} + +//IsCurrent returns true if the node is currently +// synced up with the network +func (s *Syncmgr) IsCurrent() bool { + return s.syncmode == normalMode +} diff --git a/pkg/syncmgr/syncmgr_onblock_test.go b/pkg/syncmgr/syncmgr_onblock_test.go new file mode 100644 index 000000000..d5b79f0b3 --- /dev/null +++ b/pkg/syncmgr/syncmgr_onblock_test.go @@ -0,0 +1,97 @@ +package syncmgr + +import ( + "testing" + + "github.com/CityOfZion/neo-go/pkg/chain" + + "github.com/CityOfZion/neo-go/pkg/wire/payload" + "github.com/stretchr/testify/assert" +) + +func TestHeadersModeOnBlock(t *testing.T) { + + syncmgr, helper := setupSyncMgr(headersMode) + + syncmgr.OnBlock(&mockPeer{}, randomBlockMessage(t, 0)) + + // In headerMode, we do nothing + assert.Equal(t, 0, helper.blocksProcessed) +} + +func TestBlockModeOnBlock(t *testing.T) { + + syncmgr, helper := setupSyncMgr(blockMode) + + syncmgr.OnBlock(&mockPeer{}, randomBlockMessage(t, 0)) + + // When a block is received in blockMode, it is processed + assert.Equal(t, 1, helper.blocksProcessed) +} +func TestNormalModeOnBlock(t *testing.T) { + + syncmgr, helper := setupSyncMgr(normalMode) + + syncmgr.OnBlock(&mockPeer{}, randomBlockMessage(t, 0)) + + // When a block is received in normal, it is processed + assert.Equal(t, 1, helper.blocksProcessed) +} + +func TestBlockModeToNormalMode(t *testing.T) { + + syncmgr, _ := setupSyncMgr(blockMode) + + peer := &mockPeer{ + height: 100, + } + + blkMessage := randomBlockMessage(t, 100) + + syncmgr.OnBlock(peer, blkMessage) + + // We should switch to normal mode, since the block + //we received is close to the height of the peer. See cruiseHeight + assert.Equal(t, normalMode, syncmgr.syncmode) + +} +func TestBlockModeStayInBlockMode(t *testing.T) { + + syncmgr, _ := setupSyncMgr(blockMode) + + // We need our latest know hash to not be equal to the hash + // of the block we received, to stay in blockmode + syncmgr.headerHash = randomUint256(t) + + peer := &mockPeer{ + height: 2000, + } + + blkMessage := randomBlockMessage(t, 100) + + syncmgr.OnBlock(peer, blkMessage) + + // We should stay in block mode, since the block we received is + // still quite far behind the peers height + assert.Equal(t, blockMode, syncmgr.syncmode) +} +func TestBlockModeAlreadyExistsErr(t *testing.T) { + + syncmgr, helper := setupSyncMgr(blockMode) + helper.err = chain.ErrBlockAlreadyExists + + syncmgr.OnBlock(&mockPeer{}, randomBlockMessage(t, 100)) + + assert.Equal(t, 0, helper.blockFetchRequest) + + // If we have a block already exists in blockmode, then we + // switch back to headers mode. + assert.Equal(t, headersMode, syncmgr.syncmode) +} + +func randomBlockMessage(t *testing.T, height uint32) *payload.BlockMessage { + blockMessage, err := payload.NewBlockMessage() + blockMessage.BlockBase.Index = height + assert.Nil(t, err) + return blockMessage +} diff --git a/pkg/syncmgr/syncmgr_onheaders_test.go b/pkg/syncmgr/syncmgr_onheaders_test.go new file mode 100644 index 000000000..2f60a4b72 --- /dev/null +++ b/pkg/syncmgr/syncmgr_onheaders_test.go @@ -0,0 +1,117 @@ +package syncmgr + +import ( + "testing" + + "github.com/CityOfZion/neo-go/pkg/chain" + + "github.com/stretchr/testify/assert" + + "github.com/CityOfZion/neo-go/pkg/wire/util" +) + +func TestHeadersModeOnHeaders(t *testing.T) { + + syncmgr, helper := setupSyncMgr(headersMode) + + syncmgr.OnHeader(&mockPeer{}, randomHeadersMessage(t, 0)) + + // Since there were no headers, we should have exited early and processed nothing + assert.Equal(t, 0, helper.headersProcessed) + + // ProcessHeaders should have been called once to process all 100 headers + syncmgr.OnHeader(&mockPeer{}, randomHeadersMessage(t, 100)) + assert.Equal(t, 100, helper.headersProcessed) + + // Mode should now be blockMode + assert.Equal(t, blockMode, syncmgr.syncmode) + +} + +func TestBlockModeOnHeaders(t *testing.T) { + syncmgr, helper := setupSyncMgr(blockMode) + + // If we receive a header in blockmode, no headers will be processed + syncmgr.OnHeader(&mockPeer{}, randomHeadersMessage(t, 100)) + assert.Equal(t, 0, helper.headersProcessed) +} +func TestNormalModeOnHeadersMaxHeaders(t *testing.T) { + syncmgr, helper := setupSyncMgr(normalMode) + + // If we receive a header in normalmode, headers will be processed + syncmgr.OnHeader(&mockPeer{}, randomHeadersMessage(t, 2000)) + assert.Equal(t, 2000, helper.headersProcessed) + + // Mode should now be headersMode since we received 2000 headers + assert.Equal(t, headersMode, syncmgr.syncmode) +} + +// This differs from the previous function in that +//we did not receive the max amount of headers +func TestNormalModeOnHeaders(t *testing.T) { + syncmgr, helper := setupSyncMgr(normalMode) + + // If we receive a header in normalmode, headers will be processed + syncmgr.OnHeader(&mockPeer{}, randomHeadersMessage(t, 200)) + assert.Equal(t, 200, helper.headersProcessed) + + // Because we did not receive 2000 headers, we switch to blockMode + assert.Equal(t, blockMode, syncmgr.syncmode) +} + +func TestLastHeaderUpdates(t *testing.T) { + syncmgr, _ := setupSyncMgr(headersMode) + + hdrsMessage := randomHeadersMessage(t, 200) + hdrs := hdrsMessage.Headers + lastHeader := hdrs[len(hdrs)-1] + + syncmgr.OnHeader(&mockPeer{}, hdrsMessage) + + // Headers are processed in headersMode + // Last header should be updated + assert.True(t, syncmgr.headerHash.Equals(lastHeader.Hash)) + + // Change mode to blockMode and reset lastHeader + syncmgr.syncmode = blockMode + syncmgr.headerHash = util.Uint256{} + + syncmgr.OnHeader(&mockPeer{}, hdrsMessage) + + // header should not be changed + assert.False(t, syncmgr.headerHash.Equals(lastHeader.Hash)) + + // Change mode to normalMode and reset lastHeader + syncmgr.syncmode = normalMode + syncmgr.headerHash = util.Uint256{} + + syncmgr.OnHeader(&mockPeer{}, hdrsMessage) + + // headers are processed in normalMode + // hash should be updated + assert.True(t, syncmgr.headerHash.Equals(lastHeader.Hash)) + +} + +func TestHeadersModeOnHeadersErr(t *testing.T) { + + syncmgr, helper := setupSyncMgr(headersMode) + helper.err = &chain.ValidationError{} + + syncmgr.OnHeader(&mockPeer{}, randomHeadersMessage(t, 200)) + + // On a validation error, we should request for another peer + // to send us these headers + assert.Equal(t, 1, helper.headersFetchRequest) +} + +func TestNormalModeOnHeadersErr(t *testing.T) { + syncmgr, helper := setupSyncMgr(normalMode) + helper.err = &chain.ValidationError{} + + syncmgr.OnHeader(&mockPeer{}, randomHeadersMessage(t, 200)) + + // On a validation error, we should request for another peer + // to send us these headers + assert.Equal(t, 1, helper.headersFetchRequest) +} From 1a6bdd40993ab19e735f9fc00772baf246d1ade9 Mon Sep 17 00:00:00 2001 From: decentralisedkev <37423678+decentralisedkev@users.noreply.github.com> Date: Thu, 28 Mar 2019 22:49:34 +0000 Subject: [PATCH 14/24] [Server] Implements Orchestration server (#252) * [pubsub] - remove pubsub package * [chain] - Add height to chain * [peer] - remove unnecesary println * [server] - Implement server package * Add main.go to run node --- main.go | 20 +++++++ pkg/chain/chain.go | 9 +++- pkg/peer/peer.go | 1 - pkg/pubsub/event.go | 20 ------- pkg/pubsub/pub.go | 21 -------- pkg/pubsub/sub.go | 7 --- pkg/server/addrmgr.go | 7 +++ pkg/server/chain.go | 15 ++++++ pkg/server/connmgr.go | 62 +++++++++++++++++++++ pkg/server/database.go | 14 +++++ pkg/server/peerconfig.go | 23 ++++++++ pkg/server/peermgr.go | 9 ++++ pkg/server/server.go | 113 +++++++++++++++++++++++++++++++++++++++ pkg/server/syncmgr.go | 94 ++++++++++++++++++++++++++++++++ pkg/syncmgr/config.go | 2 +- 15 files changed, 366 insertions(+), 51 deletions(-) create mode 100644 main.go delete mode 100644 pkg/pubsub/event.go delete mode 100644 pkg/pubsub/pub.go delete mode 100644 pkg/pubsub/sub.go create mode 100644 pkg/server/addrmgr.go create mode 100644 pkg/server/chain.go create mode 100644 pkg/server/connmgr.go create mode 100644 pkg/server/database.go create mode 100644 pkg/server/peerconfig.go create mode 100644 pkg/server/peermgr.go create mode 100644 pkg/server/server.go create mode 100644 pkg/server/syncmgr.go diff --git a/main.go b/main.go new file mode 100644 index 000000000..b7b54ab7b --- /dev/null +++ b/main.go @@ -0,0 +1,20 @@ +package main + +import ( + "fmt" + + "github.com/CityOfZion/neo-go/pkg/server" + "github.com/CityOfZion/neo-go/pkg/wire/protocol" +) + +func main() { + s, err := server.New(protocol.MainNet, 10332) + if err != nil { + fmt.Println(err) + return + } + err = s.Run() + if err != nil { + fmt.Println("Server has stopped from the following error: ", err.Error()) + } +} diff --git a/pkg/chain/chain.go b/pkg/chain/chain.go index c44764388..f49aedbac 100644 --- a/pkg/chain/chain.go +++ b/pkg/chain/chain.go @@ -23,7 +23,8 @@ var ( // Chain represents a blockchain instance type Chain struct { - Db *Chaindb + Db *Chaindb + height uint32 } // New returns a new chain instance @@ -128,3 +129,9 @@ func (c *Chain) ProcessHeaders(hdrs []*payload.BlockBase) error { func (c *Chain) verifyHeaders(hdrs []*payload.BlockBase) error { return nil } + +// CurrentHeight returns the index of the block +// at the tip of the chain +func (c Chain) CurrentHeight() uint32 { + return c.height +} diff --git a/pkg/peer/peer.go b/pkg/peer/peer.go index d1594d6de..8fe2f28d6 100644 --- a/pkg/peer/peer.go +++ b/pkg/peer/peer.go @@ -176,7 +176,6 @@ func (p *Peer) IsVerackReceived() bool { //NotifyDisconnect returns once the peer has disconnected // Blocking func (p *Peer) NotifyDisconnect() { - fmt.Println("Peer has not disconnected yet") <-p.quitch fmt.Println("Peer has just disconnected") } diff --git a/pkg/pubsub/event.go b/pkg/pubsub/event.go deleted file mode 100644 index 1a57e86cd..000000000 --- a/pkg/pubsub/event.go +++ /dev/null @@ -1,20 +0,0 @@ -package pubsub - -// EventType is an enum -// representing the types of messages we can subscribe to -type EventType int - -const ( - // NewBlock is called When blockchain connects a new block, it will emit an NewBlock Event - NewBlock EventType = iota - // BadBlock is called When blockchain declines a block, it will emit a new block event - BadBlock - // BadHeader is called When blockchain rejects a Header, it will emit this event - BadHeader -) - -// Event represents a new Event that a subscriber can listen to -type Event struct { - Type EventType // E.g. event.NewBlock - data []byte // Raw information -} diff --git a/pkg/pubsub/pub.go b/pkg/pubsub/pub.go deleted file mode 100644 index a9f6084a1..000000000 --- a/pkg/pubsub/pub.go +++ /dev/null @@ -1,21 +0,0 @@ -package pubsub - -// Publisher sends events to subscribers -type Publisher struct { - subs []Subscriber -} - -// Send iterates over each subscriber and checks -// if they are interested in the Event -// By looking at their topics, if they are then -// the event is emitted to them -func (p *Publisher) Send(e Event) error { - for _, sub := range p.subs { - for _, topic := range sub.Topics() { - if e.Type == topic { - sub.Emit(e) - } - } - } - return nil -} diff --git a/pkg/pubsub/sub.go b/pkg/pubsub/sub.go deleted file mode 100644 index 4540870ff..000000000 --- a/pkg/pubsub/sub.go +++ /dev/null @@ -1,7 +0,0 @@ -package pubsub - -// Subscriber will listen for Events from publishers -type Subscriber interface { - Topics() []EventType - Emit(Event) -} diff --git a/pkg/server/addrmgr.go b/pkg/server/addrmgr.go new file mode 100644 index 000000000..6237d79e3 --- /dev/null +++ b/pkg/server/addrmgr.go @@ -0,0 +1,7 @@ +package server + +// etAddress will return a viable address to connect to +// Currently it is hardcoded to be one neo node until address manager is implemented +func (s *Server) getAddress() (string, error) { + return "seed1.ngd.network:10333", nil +} diff --git a/pkg/server/chain.go b/pkg/server/chain.go new file mode 100644 index 000000000..23b6c2cdc --- /dev/null +++ b/pkg/server/chain.go @@ -0,0 +1,15 @@ +package server + +import ( + "github.com/CityOfZion/neo-go/pkg/chain" + "github.com/CityOfZion/neo-go/pkg/database" + "github.com/CityOfZion/neo-go/pkg/wire/protocol" +) + +func setupChain(db database.Database, net protocol.Magic) (*chain.Chain, error) { + chain, err := chain.New(db, net) + if err != nil { + return nil, err + } + return chain, nil +} diff --git a/pkg/server/connmgr.go b/pkg/server/connmgr.go new file mode 100644 index 000000000..64b16bdaf --- /dev/null +++ b/pkg/server/connmgr.go @@ -0,0 +1,62 @@ +package server + +import ( + "encoding/hex" + "fmt" + "net" + "strconv" + + "github.com/CityOfZion/neo-go/pkg/connmgr" + + "github.com/CityOfZion/neo-go/pkg/peer" + "github.com/CityOfZion/neo-go/pkg/wire/util" + iputils "github.com/CityOfZion/neo-go/pkg/wire/util/ip" +) + +func setupConnManager(s *Server, port uint16) *connmgr.Connmgr { + cfg := connmgr.Config{ + GetAddress: s.getAddress, + OnAccept: s.onAccept, + OnConnection: s.onConnection, + AddressPort: iputils.GetLocalIP().String() + ":" + strconv.FormatUint(uint64(port), 10), + } + return connmgr.New(cfg) +} + +func (s *Server) onConnection(conn net.Conn, addr string) { + fmt.Println("We have connected successfully to: ", addr) + + p := peer.NewPeer(conn, false, *s.peerCfg) + err := p.Run() + if err != nil { + fmt.Println("Error running peer" + err.Error()) + return + } + + s.pmg.AddPeer(p) + + byt, err := hex.DecodeString("d42561e3d30e15be6400b6df2f328e02d2bf6354c41dce433bc57687c82144bf") + if err != nil { + fmt.Println("Error getting hash " + err.Error()) + } + lh, err := util.Uint256DecodeBytes(byt) + if err != nil { + fmt.Println("Error getting hash " + err.Error()) + } + err = p.RequestHeaders(lh.Reverse()) + if err != nil { + fmt.Println(err) + } +} + +func (s *Server) onAccept(conn net.Conn) { + fmt.Println("A peer with address: ", conn.RemoteAddr().String(), "has connect to us") + + p := peer.NewPeer(conn, true, *s.peerCfg) + err := p.Run() + if err != nil { + fmt.Println("Error running peer" + err.Error()) + return + } + s.pmg.AddPeer(p) +} diff --git a/pkg/server/database.go b/pkg/server/database.go new file mode 100644 index 000000000..9b7eea9f0 --- /dev/null +++ b/pkg/server/database.go @@ -0,0 +1,14 @@ +package server + +import ( + "github.com/CityOfZion/neo-go/pkg/database" + "github.com/CityOfZion/neo-go/pkg/wire/protocol" +) + +func setupDatabase(net protocol.Magic) (database.Database, error) { + db, err := database.New(net.String()) + if err != nil { + return nil, err + } + return db, nil +} diff --git a/pkg/server/peerconfig.go b/pkg/server/peerconfig.go new file mode 100644 index 000000000..4fa8307bf --- /dev/null +++ b/pkg/server/peerconfig.go @@ -0,0 +1,23 @@ +package server + +import ( + "math/rand" + + "github.com/CityOfZion/neo-go/pkg/peer" + "github.com/CityOfZion/neo-go/pkg/wire/protocol" +) + +func setupPeerConfig(s *Server, port uint16, net protocol.Magic) *peer.LocalConfig { + return &peer.LocalConfig{ + Net: net, + UserAgent: "NEO-GO", + Services: protocol.NodePeerService, + Nonce: rand.Uint32(), + ProtocolVer: 0, + Relay: false, + Port: port, + StartHeight: s.chain.CurrentHeight, + OnHeader: s.onHeader, + OnBlock: s.onBlock, + } +} diff --git a/pkg/server/peermgr.go b/pkg/server/peermgr.go new file mode 100644 index 000000000..670be60fc --- /dev/null +++ b/pkg/server/peermgr.go @@ -0,0 +1,9 @@ +package server + +import ( + "github.com/CityOfZion/neo-go/pkg/peermgr" +) + +func setupPeerManager() *peermgr.PeerMgr { + return peermgr.New() +} diff --git a/pkg/server/server.go b/pkg/server/server.go new file mode 100644 index 000000000..b1e4c0f5b --- /dev/null +++ b/pkg/server/server.go @@ -0,0 +1,113 @@ +package server + +import ( + "fmt" + + "github.com/CityOfZion/neo-go/pkg/peermgr" + + "github.com/CityOfZion/neo-go/pkg/chain" + "github.com/CityOfZion/neo-go/pkg/connmgr" + "github.com/CityOfZion/neo-go/pkg/peer" + "github.com/CityOfZion/neo-go/pkg/syncmgr" + + "github.com/CityOfZion/neo-go/pkg/database" + + "github.com/CityOfZion/neo-go/pkg/wire/protocol" +) + +// Server orchestrates all of the modules +type Server struct { + net protocol.Magic + stopCh chan error + + // Modules + db database.Database + smg *syncmgr.Syncmgr + cmg *connmgr.Connmgr + pmg *peermgr.PeerMgr + chain *chain.Chain + + peerCfg *peer.LocalConfig +} + +//New creates a new server object for a particular network and sets up each module +func New(net protocol.Magic, port uint16) (*Server, error) { + s := &Server{ + net: net, + stopCh: make(chan error, 0), + } + + // Setup database + db, err := setupDatabase(net) + if err != nil { + return nil, err + } + s.db = db + + // setup peermgr + peermgr := setupPeerManager() + s.pmg = peermgr + + // Setup chain + chain, err := setupChain(db, net) + if err != nil { + return nil, err + } + s.chain = chain + + // Setup sync manager + syncmgr := setupSyncManager(s) + s.smg = syncmgr + + // Setup connection manager + connmgr := setupConnManager(s, port) + s.cmg = connmgr + + // Setup peer config + peerCfg := setupPeerConfig(s, port, net) + s.peerCfg = peerCfg + + return s, nil +} + +// Run starts the daemon by connecting to previously nodes or connectng to seed nodes. +// This should be called once all modules have been setup +func (s *Server) Run() error { + fmt.Println("Server is starting up") + + // start the connmgr + err := s.cmg.Run() + if err != nil { + return err + } + + // Attempt to connect to a peer + err = s.cmg.NewRequest() + if err != nil { + return err + } + + // Request header to start synchronisation + bestHeader, err := s.chain.Db.GetLastHeader() + if err != nil { + return err + } + err = s.pmg.RequestHeaders(bestHeader.Hash.Reverse()) + if err != nil { + return err + } + fmt.Println("Server Successfully started") + return s.wait() +} + +func (s *Server) wait() error { + err := <-s.stopCh + return err +} + +// Stop stops the server +func (s *Server) Stop(err error) error { + fmt.Println("Server is shutting down") + s.stopCh <- err + return nil +} diff --git a/pkg/server/syncmgr.go b/pkg/server/syncmgr.go new file mode 100644 index 000000000..3df456626 --- /dev/null +++ b/pkg/server/syncmgr.go @@ -0,0 +1,94 @@ +package server + +import ( + "encoding/binary" + + "github.com/CityOfZion/neo-go/pkg/peer" + "github.com/CityOfZion/neo-go/pkg/syncmgr" + "github.com/CityOfZion/neo-go/pkg/wire/payload" + "github.com/CityOfZion/neo-go/pkg/wire/util" +) + +func setupSyncManager(s *Server) *syncmgr.Syncmgr { + + cfg := &syncmgr.Config{ + ProcessBlock: s.processBlock, + ProcessHeaders: s.processHeaders, + + RequestBlock: s.requestBlock, + RequestHeaders: s.requestHeaders, + + GetNextBlockHash: s.getNextBlockHash, + AskForNewBlocks: s.askForNewBlocks, + + FetchHeadersAgain: s.fetchHeadersAgain, + FetchBlockAgain: s.fetchBlockAgain, + } + + return syncmgr.New(cfg) +} + +func (s *Server) onHeader(peer *peer.Peer, hdrsMessage *payload.HeadersMessage) { + s.pmg.MsgReceived(peer, hdrsMessage.Command()) + s.smg.OnHeader(peer, hdrsMessage) +} + +func (s *Server) onBlock(peer *peer.Peer, blockMsg *payload.BlockMessage) { + s.pmg.MsgReceived(peer, blockMsg.Command()) + s.smg.OnBlock(peer, blockMsg) +} + +func (s *Server) processBlock(block payload.Block) error { + return s.chain.ProcessBlock(block) +} + +func (s *Server) processHeaders(hdrs []*payload.BlockBase) error { + return s.chain.ProcessHeaders(hdrs) +} + +func (s *Server) requestHeaders(hash util.Uint256) error { + return s.pmg.RequestHeaders(hash) +} + +func (s *Server) requestBlock(hash util.Uint256) error { + return s.pmg.RequestBlock(hash) +} + +// getNextBlockHash searches the database for the blockHash +// that is the height above our best block. The hash will be taken from a header. +func (s *Server) getNextBlockHash() (util.Uint256, error) { + bestBlock, err := s.chain.Db.GetLastBlock() + if err != nil { + // Panic! + // XXX: One alternative, is to get the network, erase the database and then start again from scratch. + // This should never happen. The latest block will always be atleast the genesis block + panic("could not get best block from database" + err.Error()) + } + + index := make([]byte, 4) + binary.BigEndian.PutUint32(index, bestBlock.Index+1) + + hdr, err := s.chain.Db.GetHeaderFromHeight(index) + if err != nil { + return util.Uint256{}, err + } + return hdr.Hash, nil +} + +func (s *Server) getBestBlockHash() (util.Uint256, error) { + return util.Uint256{}, nil +} + +func (s *Server) askForNewBlocks() { + // send a getblocks message with the latest block saved + + // when we receive something then send get data +} + +func (s *Server) fetchHeadersAgain(util.Uint256) error { + return nil +} + +func (s *Server) fetchBlockAgain(util.Uint256) error { + return nil +} diff --git a/pkg/syncmgr/config.go b/pkg/syncmgr/config.go index bc921b33b..af27f37c6 100644 --- a/pkg/syncmgr/config.go +++ b/pkg/syncmgr/config.go @@ -9,7 +9,7 @@ import ( type Config struct { // Chain functions - ProcessBlock func(msg payload.Block) error + ProcessBlock func(block payload.Block) error ProcessHeaders func(hdrs []*payload.BlockBase) error // RequestHeaders will send a getHeaders request From abb4da9cbd615c062979cf544ba1d1c891176160 Mon Sep 17 00:00:00 2001 From: decentralisedkev <37423678+decentralisedkev@users.noreply.github.com> Date: Sat, 30 Mar 2019 18:10:27 +0000 Subject: [PATCH 15/24] [PeerMgr] Add Caching and Re-processing system (#263) [peermgr] - Add request cache with tests - Add requestCache to peermgr - refactored peer manager tests - Added blockInfo struct, to allow sorting on the blockIndex - added helper methods for cache, pickItem, pickFirstItem, removeHash, findHash and refactored tests - renamed requestcache to blockcache - refactored peer manager to use block cache for block requests *only* - added blockCallPeer function to handle block requests only - refactored onDisconnect to add back any pending peer requests that the disconnected peer did not complete into the peer manager queue [peermgr/server] - Modify onBlock handler in server, to send peermgr a BlockInfo struct [peermgr/syncmgr/server] - Modified blockIndex in BlockInfo to be uint32 and not uint64 - RequestBlocks in syncmgr now takes an index along with the hash - modified syncmgr code to pass index along with hash in all methods --- pkg/peermgr/blockcache.go | 155 ++++++++++++++++++++++++++++++++ pkg/peermgr/blockcache_test.go | 80 +++++++++++++++++ pkg/peermgr/peermgr.go | 125 +++++++++++++++++++++++--- pkg/peermgr/peermgr_test.go | 56 ++++++++---- pkg/server/syncmgr.go | 14 ++- pkg/syncmgr/blockmode.go | 2 +- pkg/syncmgr/config.go | 2 +- pkg/syncmgr/headermode.go | 3 +- pkg/syncmgr/mockhelpers_test.go | 2 +- pkg/syncmgr/normalmode.go | 3 +- 10 files changed, 407 insertions(+), 35 deletions(-) create mode 100644 pkg/peermgr/blockcache.go create mode 100644 pkg/peermgr/blockcache_test.go diff --git a/pkg/peermgr/blockcache.go b/pkg/peermgr/blockcache.go new file mode 100644 index 000000000..8e06b8251 --- /dev/null +++ b/pkg/peermgr/blockcache.go @@ -0,0 +1,155 @@ +package peermgr + +import ( + "errors" + "sort" + "sync" + + "github.com/CityOfZion/neo-go/pkg/wire/util" +) + +var ( + //ErrCacheLimit is returned when the cache limit is reached + ErrCacheLimit = errors.New("nomore items can be added to the cache") + + //ErrNoItems is returned when pickItem is called and there are no items in the cache + ErrNoItems = errors.New("there are no items in the cache") + + //ErrDuplicateItem is returned when you try to add the same item, more than once to the cache + ErrDuplicateItem = errors.New("this item is already in the cache") +) + +//BlockInfo holds the necessary information that the cache needs +// to sort and store block requests +type BlockInfo struct { + BlockHash util.Uint256 + BlockIndex uint32 +} + +// Equals returns true if two blockInfo objects +// have the same hash and the same index +func (bi *BlockInfo) Equals(other BlockInfo) bool { + return bi.BlockHash.Equals(other.BlockHash) && bi.BlockIndex == other.BlockIndex +} + +// indexSorter sorts the blockInfos by blockIndex. +type indexSorter []BlockInfo + +func (is indexSorter) Len() int { return len(is) } +func (is indexSorter) Swap(i, j int) { is[i], is[j] = is[j], is[i] } +func (is indexSorter) Less(i, j int) bool { return is[i].BlockIndex < is[j].BlockIndex } + +//blockCache will cache any pending block requests +// for the node when there are no available nodes +type blockCache struct { + cacheLimit int + cacheLock sync.Mutex + cache []BlockInfo +} + +func newBlockCache(cacheLimit int) *blockCache { + return &blockCache{ + cache: make([]BlockInfo, 0, cacheLimit), + cacheLimit: cacheLimit, + } +} + +func (bc *blockCache) addBlockInfo(bi BlockInfo) error { + if bc.cacheLen() == bc.cacheLimit { + return ErrCacheLimit + } + + bc.cacheLock.Lock() + defer bc.cacheLock.Unlock() + + // Check for duplicates. slice will always be small so a simple for loop will work + for _, bInfo := range bc.cache { + if bInfo.Equals(bi) { + return ErrDuplicateItem + } + } + bc.cache = append(bc.cache, bi) + + sort.Sort(indexSorter(bc.cache)) + + return nil +} + +func (bc *blockCache) addBlockInfos(bis []BlockInfo) error { + + if len(bis)+bc.cacheLen() > bc.cacheLimit { + return errors.New("too many items to add, this will exceed the cache limit") + } + + for _, bi := range bis { + err := bc.addBlockInfo(bi) + if err != nil { + return err + } + } + return nil +} + +func (bc *blockCache) cacheLen() int { + bc.cacheLock.Lock() + defer bc.cacheLock.Unlock() + return len(bc.cache) +} + +func (bc *blockCache) pickFirstItem() (BlockInfo, error) { + return bc.pickItem(0) +} + +func (bc *blockCache) pickAllItems() ([]BlockInfo, error) { + + numOfItems := bc.cacheLen() + + items := make([]BlockInfo, 0, numOfItems) + + for i := 0; i < numOfItems; i++ { + bi, err := bc.pickFirstItem() + if err != nil { + return nil, err + } + items = append(items, bi) + } + return items, nil +} + +func (bc *blockCache) pickItem(i uint) (BlockInfo, error) { + if bc.cacheLen() < 1 { + return BlockInfo{}, ErrNoItems + } + + if i >= uint(bc.cacheLen()) { + return BlockInfo{}, errors.New("index out of range") + } + + bc.cacheLock.Lock() + defer bc.cacheLock.Unlock() + + item := bc.cache[i] + bc.cache = append(bc.cache[:i], bc.cache[i+1:]...) + return item, nil +} + +func (bc *blockCache) removeHash(hashToRemove util.Uint256) error { + index, err := bc.findHash(hashToRemove) + if err != nil { + return err + } + + _, err = bc.pickItem(uint(index)) + return err +} + +func (bc *blockCache) findHash(hashToFind util.Uint256) (int, error) { + bc.cacheLock.Lock() + defer bc.cacheLock.Unlock() + for i, bInfo := range bc.cache { + if bInfo.BlockHash.Equals(hashToFind) { + return i, nil + } + } + return -1, errors.New("hash cannot be found in the cache") +} diff --git a/pkg/peermgr/blockcache_test.go b/pkg/peermgr/blockcache_test.go new file mode 100644 index 000000000..3cb928e9e --- /dev/null +++ b/pkg/peermgr/blockcache_test.go @@ -0,0 +1,80 @@ +package peermgr + +import ( + "math/rand" + "testing" + + "github.com/CityOfZion/neo-go/pkg/wire/util" + "github.com/stretchr/testify/assert" +) + +func TestAddBlock(t *testing.T) { + + bc := &blockCache{ + cacheLimit: 20, + } + bi := randomBlockInfo(t) + + err := bc.addBlockInfo(bi) + assert.Equal(t, nil, err) + + assert.Equal(t, 1, bc.cacheLen()) + + err = bc.addBlockInfo(bi) + assert.Equal(t, ErrDuplicateItem, err) + + assert.Equal(t, 1, bc.cacheLen()) +} + +func TestCacheLimit(t *testing.T) { + + bc := &blockCache{ + cacheLimit: 20, + } + + for i := 0; i < bc.cacheLimit; i++ { + err := bc.addBlockInfo(randomBlockInfo(t)) + assert.Equal(t, nil, err) + } + + err := bc.addBlockInfo(randomBlockInfo(t)) + assert.Equal(t, ErrCacheLimit, err) + + assert.Equal(t, bc.cacheLimit, bc.cacheLen()) +} +func TestPickItem(t *testing.T) { + + bc := &blockCache{ + cacheLimit: 20, + } + + for i := 0; i < bc.cacheLimit; i++ { + err := bc.addBlockInfo(randomBlockInfo(t)) + assert.Equal(t, nil, err) + } + + for i := 0; i < bc.cacheLimit; i++ { + _, err := bc.pickFirstItem() + assert.Equal(t, nil, err) + } + + assert.Equal(t, 0, bc.cacheLen()) +} + +func randomUint256(t *testing.T) util.Uint256 { + rand32 := make([]byte, 32) + rand.Read(rand32) + + u, err := util.Uint256DecodeBytes(rand32) + assert.Equal(t, nil, err) + + return u +} + +func randomBlockInfo(t *testing.T) BlockInfo { + + return BlockInfo{ + randomUint256(t), + rand.Uint32(), + } +} diff --git a/pkg/peermgr/peermgr.go b/pkg/peermgr/peermgr.go index 047a109c3..e83e37768 100644 --- a/pkg/peermgr/peermgr.go +++ b/pkg/peermgr/peermgr.go @@ -2,6 +2,7 @@ package peermgr import ( "errors" + "fmt" "sync" "github.com/CityOfZion/neo-go/pkg/wire/command" @@ -9,6 +10,15 @@ import ( "github.com/CityOfZion/neo-go/pkg/wire/util" ) +const ( + // blockCacheLimit is the maximum amount of pending requests that the cache can hold + pendingBlockCacheLimit = 20 + + //peerBlockCacheLimit is the maximum amount of inflight blocks that a peer can + // have, before they are flagged as busy + peerBlockCacheLimit = 1 +) + var ( //ErrNoAvailablePeers is returned when a request for data from a peer is invoked // but there are no available peers to request data from @@ -28,6 +38,10 @@ type mPeer interface { } type peerstats struct { + // when a peer is sent a blockRequest + // the peermanager will track this using this blockCache + blockCache *blockCache + // all other requests will be tracked using the requests map requests map[command.Type]bool } @@ -35,12 +49,15 @@ type peerstats struct { type PeerMgr struct { pLock sync.RWMutex peers map[mPeer]peerstats + + requestCache *blockCache } //New returns a new peermgr object func New() *PeerMgr { return &PeerMgr{ - peers: make(map[mPeer]peerstats), + peers: make(map[mPeer]peerstats), + requestCache: newBlockCache(pendingBlockCacheLimit), } } @@ -52,7 +69,10 @@ func (pmgr *PeerMgr) AddPeer(peer mPeer) { if _, exists := pmgr.peers[peer]; exists { return } - pmgr.peers[peer] = peerstats{requests: make(map[command.Type]bool)} + pmgr.peers[peer] = peerstats{ + requests: make(map[command.Type]bool), + blockCache: newBlockCache(peerBlockCacheLimit), + } go pmgr.onDisconnect(peer) } @@ -61,6 +81,8 @@ func (pmgr *PeerMgr) AddPeer(peer mPeer) { func (pmgr *PeerMgr) MsgReceived(peer mPeer, cmd command.Type) error { pmgr.pLock.Lock() defer pmgr.pLock.Unlock() + + // if peer was unknown then disconnect val, ok := pmgr.peers[peer] if !ok { @@ -76,6 +98,44 @@ func (pmgr *PeerMgr) MsgReceived(peer mPeer, cmd command.Type) error { return nil } +//BlockMsgReceived notifies the peer manager that we have received a +// block message from a peer +func (pmgr *PeerMgr) BlockMsgReceived(peer mPeer, bi BlockInfo) error { + + // if peer was unknown then disconnect + val, ok := pmgr.peers[peer] + if !ok { + + go func() { + peer.NotifyDisconnect() + }() + + peer.Disconnect() + return ErrUnknownPeer + } + + // // remove item from the peersBlock cache + err := val.blockCache.removeHash(bi.BlockHash) + if err != nil { + return err + } + + // check if cache empty, if so then return + if pmgr.requestCache.cacheLen() == 0 { + return nil + } + + // Try to clean an item from the pendingBlockCache, a peer has just finished serving a block request + cachedBInfo, err := pmgr.requestCache.pickFirstItem() + if err != nil { + return err + } + + return pmgr.blockCallPeer(cachedBInfo, func(p mPeer) error { + return p.RequestBlocks([]util.Uint256{cachedBInfo.BlockHash}) + }) +} + // Len returns the amount of peers that the peer manager //currently knows about func (pmgr *PeerMgr) Len() int { @@ -84,25 +144,34 @@ func (pmgr *PeerMgr) Len() int { return len(pmgr.peers) } -// RequestBlock will request a block from the most +// RequestBlock will request a block from the most // available peer. Then update it's stats, so we know that // this peer is busy -func (pmgr *PeerMgr) RequestBlock(hash util.Uint256) error { - return pmgr.callPeerForCmd(command.Block, func(p mPeer) error { - return p.RequestBlocks([]util.Uint256{hash}) +func (pmgr *PeerMgr) RequestBlock(bi BlockInfo) error { + pmgr.pLock.Lock() + defer pmgr.pLock.Unlock() + + err := pmgr.blockCallPeer(bi, func(p mPeer) error { + return p.RequestBlocks([]util.Uint256{bi.BlockHash}) }) + + if err == ErrNoAvailablePeers { + return pmgr.requestCache.addBlockInfo(bi) + } + + return err } // RequestHeaders will request a headers from the most available peer. func (pmgr *PeerMgr) RequestHeaders(hash util.Uint256) error { + pmgr.pLock.Lock() + defer pmgr.pLock.Unlock() return pmgr.callPeerForCmd(command.Headers, func(p mPeer) error { return p.RequestHeaders(hash) }) } func (pmgr *PeerMgr) callPeerForCmd(cmd command.Type, f func(p mPeer) error) error { - pmgr.pLock.Lock() - defer pmgr.pLock.Unlock() for peer, stats := range pmgr.peers { if !stats.requests[cmd] { stats.requests[cmd] = true @@ -111,12 +180,48 @@ func (pmgr *PeerMgr) callPeerForCmd(cmd command.Type, f func(p mPeer) error) err } return ErrNoAvailablePeers } + +func (pmgr *PeerMgr) blockCallPeer(bi BlockInfo, f func(p mPeer) error) error { + for peer, stats := range pmgr.peers { + if stats.blockCache.cacheLen() < peerBlockCacheLimit { + err := stats.blockCache.addBlockInfo(bi) + if err != nil { + return err + } + return f(peer) + } + } + return ErrNoAvailablePeers +} + func (pmgr *PeerMgr) onDisconnect(p mPeer) { // Blocking until peer is disconnected p.NotifyDisconnect() pmgr.pLock.Lock() - delete(pmgr.peers, p) - pmgr.pLock.Unlock() + defer func() { + delete(pmgr.peers, p) + pmgr.pLock.Unlock() + }() + + // Add all of peers outstanding block requests into + // the peer managers pendingBlockRequestCache + + val, ok := pmgr.peers[p] + if !ok { + return + } + + pendingRequests, err := val.blockCache.pickAllItems() + if err != nil { + fmt.Println(err.Error()) + return + } + + err = pmgr.requestCache.addBlockInfos(pendingRequests) + if err != nil { + fmt.Println(err.Error()) + return + } } diff --git a/pkg/peermgr/peermgr_test.go b/pkg/peermgr/peermgr_test.go index 86d5be41c..9a725af43 100644 --- a/pkg/peermgr/peermgr_test.go +++ b/pkg/peermgr/peermgr_test.go @@ -4,7 +4,6 @@ import ( "testing" "github.com/CityOfZion/neo-go/pkg/wire/command" - "github.com/CityOfZion/neo-go/pkg/wire/util" "github.com/stretchr/testify/assert" ) @@ -59,42 +58,65 @@ func TestRequestBlocks(t *testing.T) { pmgr.AddPeer(peerB) pmgr.AddPeer(peerC) - err := pmgr.RequestBlock(util.Uint256{}) + firstBlock := randomBlockInfo(t) + err := pmgr.RequestBlock(firstBlock) assert.Nil(t, err) - err = pmgr.RequestBlock(util.Uint256{}) + secondBlock := randomBlockInfo(t) + err = pmgr.RequestBlock(secondBlock) assert.Nil(t, err) - err = pmgr.RequestBlock(util.Uint256{}) + thirdBlock := randomBlockInfo(t) + err = pmgr.RequestBlock(thirdBlock) assert.Nil(t, err) // Since the peer manager did not get a MsgReceived // in between the block requests // a request should be sent to all peers + // This is only true, if peerBlockCacheLimit == 1 assert.Equal(t, 1, peerA.blockRequested) assert.Equal(t, 1, peerB.blockRequested) assert.Equal(t, 1, peerC.blockRequested) // Since the peer manager still has not received a MsgReceived - // another call to request blocks, will return a NoAvailablePeerError + // another call to request blocks, will add the request to the cache + // and return a nil err - err = pmgr.RequestBlock(util.Uint256{}) - assert.Equal(t, ErrNoAvailablePeers, err) + fourthBlock := randomBlockInfo(t) + err = pmgr.RequestBlock(fourthBlock) + assert.Equal(t, nil, err) + assert.Equal(t, 1, pmgr.requestCache.cacheLen()) - // If we tell the peer manager that peerA has given us a block - // then send another BlockRequest. It will go to peerA - // since the other two peers are still busy with their - // block requests + // If we tell the peer manager that we have received a block + // it will check the cache for any pending requests and send a block request if there are any. + // The request will go to the peer who sent back the block corresponding to the first hash + // since the other two peers are still busy with their block requests - pmgr.MsgReceived(peerA, command.Block) - err = pmgr.RequestBlock(util.Uint256{}) + peer := findPeerwithHash(t, pmgr, firstBlock.BlockHash) + err = pmgr.BlockMsgReceived(peer, firstBlock) assert.Nil(t, err) - assert.Equal(t, 2, peerA.blockRequested) - assert.Equal(t, 1, peerB.blockRequested) - assert.Equal(t, 1, peerC.blockRequested) + totalRequests := peerA.blockRequested + peerB.blockRequested + peerC.blockRequested + assert.Equal(t, 4, totalRequests) + + // // cache should be empty now + assert.Equal(t, 0, pmgr.requestCache.cacheLen()) } + +// The peer manager does not tell you what peer was sent a particular block request +// For testing purposes, the following function will find that peer +func findPeerwithHash(t *testing.T, pmgr *PeerMgr, blockHash util.Uint256) mPeer { + for peer, stats := range pmgr.peers { + _, err := stats.blockCache.findHash(blockHash) + if err == nil { + return peer + } + } + assert.Fail(t, "cannot find a peer with that hash") + return nil +} + func TestRequestHeaders(t *testing.T) { pmgr := New() @@ -152,7 +174,7 @@ func TestUnknownPeer(t *testing.T) { quit: make(chan bool), } - err := pmgr.MsgReceived(unknownPeer, command.Block) + err := pmgr.MsgReceived(unknownPeer, command.Headers) assert.Equal(t, true, unknownPeer.disconnected) assert.Equal(t, ErrUnknownPeer, err) } diff --git a/pkg/server/syncmgr.go b/pkg/server/syncmgr.go index 3df456626..7de87188b 100644 --- a/pkg/server/syncmgr.go +++ b/pkg/server/syncmgr.go @@ -3,6 +3,8 @@ package server import ( "encoding/binary" + "github.com/CityOfZion/neo-go/pkg/peermgr" + "github.com/CityOfZion/neo-go/pkg/peer" "github.com/CityOfZion/neo-go/pkg/syncmgr" "github.com/CityOfZion/neo-go/pkg/wire/payload" @@ -34,7 +36,10 @@ func (s *Server) onHeader(peer *peer.Peer, hdrsMessage *payload.HeadersMessage) } func (s *Server) onBlock(peer *peer.Peer, blockMsg *payload.BlockMessage) { - s.pmg.MsgReceived(peer, blockMsg.Command()) + s.pmg.BlockMsgReceived(peer, peermgr.BlockInfo{ + BlockHash: blockMsg.Hash, + BlockIndex: blockMsg.Index, + }) s.smg.OnBlock(peer, blockMsg) } @@ -50,8 +55,11 @@ func (s *Server) requestHeaders(hash util.Uint256) error { return s.pmg.RequestHeaders(hash) } -func (s *Server) requestBlock(hash util.Uint256) error { - return s.pmg.RequestBlock(hash) +func (s *Server) requestBlock(hash util.Uint256, index uint32) error { + return s.pmg.RequestBlock(peermgr.BlockInfo{ + BlockHash: hash, + BlockIndex: index, + }) } // getNextBlockHash searches the database for the blockHash diff --git a/pkg/syncmgr/blockmode.go b/pkg/syncmgr/blockmode.go index 2e5dadd6f..83d3acd50 100644 --- a/pkg/syncmgr/blockmode.go +++ b/pkg/syncmgr/blockmode.go @@ -32,7 +32,7 @@ func (s *Syncmgr) blockModeOnBlock(peer SyncPeer, block payload.Block) error { if err != nil { return err } - err = s.cfg.RequestBlock(nextHash) + err = s.cfg.RequestBlock(nextHash, block.Index) return err } diff --git a/pkg/syncmgr/config.go b/pkg/syncmgr/config.go index af27f37c6..713059802 100644 --- a/pkg/syncmgr/config.go +++ b/pkg/syncmgr/config.go @@ -18,7 +18,7 @@ type Config struct { //RequestBlock will send a getdata request for the block // with the hash passed as a parameter - RequestBlock func(hash util.Uint256) error + RequestBlock func(hash util.Uint256, index uint32) error // GetNextBlockHash returns the block hash of the header infront of thr block // at the tip of this nodes chain. This assumes that the node is not in sync diff --git a/pkg/syncmgr/headermode.go b/pkg/syncmgr/headermode.go index 898dc933d..3a8e4d681 100644 --- a/pkg/syncmgr/headermode.go +++ b/pkg/syncmgr/headermode.go @@ -12,6 +12,7 @@ func (s *Syncmgr) headersModeOnHeaders(peer SyncPeer, hdrs []*payload.BlockBase) // Note: For the un-optimised version, we move straight to blocksOnly mode firstHash := hdrs[0].Hash + firstHdrIndex := hdrs[0].Index err := s.cfg.ProcessHeaders(hdrs) if err == nil { @@ -19,7 +20,7 @@ func (s *Syncmgr) headersModeOnHeaders(peer SyncPeer, hdrs []*payload.BlockBase) s.headerHash = hdrs[len(hdrs)-1].Hash s.syncmode = blockMode - return s.cfg.RequestBlock(firstHash) + return s.cfg.RequestBlock(firstHash, firstHdrIndex) } // Check whether it is a validation error, or a database error diff --git a/pkg/syncmgr/mockhelpers_test.go b/pkg/syncmgr/mockhelpers_test.go index 8c84225ac..157fdd513 100644 --- a/pkg/syncmgr/mockhelpers_test.go +++ b/pkg/syncmgr/mockhelpers_test.go @@ -46,7 +46,7 @@ func (s *syncTestHelper) FetchBlockAgain(util.Uint256) error { return s.err } -func (s *syncTestHelper) RequestBlock(util.Uint256) error { +func (s *syncTestHelper) RequestBlock(util.Uint256, uint32) error { s.blockFetchRequest++ return s.err } diff --git a/pkg/syncmgr/normalmode.go b/pkg/syncmgr/normalmode.go index 218d6151b..10bbac365 100644 --- a/pkg/syncmgr/normalmode.go +++ b/pkg/syncmgr/normalmode.go @@ -16,6 +16,7 @@ func (s *Syncmgr) normalModeOnHeaders(peer SyncPeer, hdrs []*payload.BlockBase) lenHeaders := len(hdrs) firstHash := hdrs[0].Hash + firstHdrIndex := hdrs[0].Index lastHash := hdrs[lenHeaders-1].Hash // Update syncmgr latest header @@ -32,7 +33,7 @@ func (s *Syncmgr) normalModeOnHeaders(peer SyncPeer, hdrs []*payload.BlockBase) // Bounds state that len > 1 && len!= 2000 & maxHeadersInMessage == 2000 // This means that we have less than 2k headers s.syncmode = blockMode - return s.cfg.RequestBlock(firstHash) + return s.cfg.RequestBlock(firstHash, firstHdrIndex) } // normalModeOnBlock is called when the sync manager is normal mode From 5ed61ff389002f94f60fbe2da7d6c2549581c1bc Mon Sep 17 00:00:00 2001 From: BlockChainDev Date: Sat, 30 Mar 2019 21:32:44 +0000 Subject: [PATCH 16/24] [syncmgr] - add blockpool plus test --- pkg/syncmgr/blockpool.go | 57 +++++++++++++++++++++++++++++++++++ pkg/syncmgr/blockpool_test.go | 42 ++++++++++++++++++++++++++ 2 files changed, 99 insertions(+) create mode 100644 pkg/syncmgr/blockpool.go create mode 100644 pkg/syncmgr/blockpool_test.go diff --git a/pkg/syncmgr/blockpool.go b/pkg/syncmgr/blockpool.go new file mode 100644 index 000000000..b038b29f3 --- /dev/null +++ b/pkg/syncmgr/blockpool.go @@ -0,0 +1,57 @@ +package syncmgr + +import ( + "sort" + + "github.com/CityOfZion/neo-go/pkg/wire/payload" +) + +func (s *Syncmgr) addToBlockPool(newBlock payload.Block) { + s.poolLock.Lock() + defer s.poolLock.Unlock() + + for _, block := range s.blockPool { + if block.Index == newBlock.Index { + return + } + } + + s.blockPool = append(s.blockPool, newBlock) + + // sort slice using block index + sort.Slice(s.blockPool, func(i, j int) bool { + return s.blockPool[i].Index < s.blockPool[j].Index + }) + +} + +func (s *Syncmgr) checkPool() error { + // Assuming that the blocks are sorted in order + + var indexesToRemove = -1 + + s.poolLock.Lock() + defer func() { + // removes all elements before this index, including the element at this index + s.blockPool = s.blockPool[indexesToRemove+1:] + s.poolLock.Unlock() + }() + + // loop iterates through the cache, processing any + // blocks that can be added to the chain + for i, block := range s.blockPool { + if s.nextBlockIndex != block.Index { + break + } + + // Save this block and save the indice so we can remove it + err := s.processBlock(block) + if err != nil { + return err + } + + indexesToRemove = i + } + + return nil +} diff --git a/pkg/syncmgr/blockpool_test.go b/pkg/syncmgr/blockpool_test.go new file mode 100644 index 000000000..c236afc2c --- /dev/null +++ b/pkg/syncmgr/blockpool_test.go @@ -0,0 +1,42 @@ +package syncmgr + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestAddBlockPoolFlush(t *testing.T) { + syncmgr, _ := setupSyncMgr(blockMode, 10) + + blockMessage := randomBlockMessage(t, 11) + + peer := &mockPeer{ + height: 100, + } + + // Since the block has Index 11 and the sync manager needs the block with index 10 + // This block will be added to the blockPool + err := syncmgr.OnBlock(peer, blockMessage) + assert.Nil(t, err) + assert.Equal(t, 1, len(syncmgr.blockPool)) + + // The sync manager is still looking for the block at height 10 + // Since this block is at height 12, it will be added to the block pool + blockMessage = randomBlockMessage(t, 12) + err = syncmgr.OnBlock(peer, blockMessage) + assert.Nil(t, err) + assert.Equal(t, 2, len(syncmgr.blockPool)) + + // This is the block that the sync manager was waiting for + // It should process this block, the check the pool for the next set of blocks + blockMessage = randomBlockMessage(t, 10) + err = syncmgr.OnBlock(peer, blockMessage) + assert.Nil(t, err) + assert.Equal(t, 0, len(syncmgr.blockPool)) + + // Since we processed 3 blocks and the sync manager started + //looking for block with index 10. The syncmananger should be looking for + // the block with index 13 + assert.Equal(t, uint32(13), syncmgr.nextBlockIndex) +} From c401247af9ed72c38db53dabe8779222be1c2fb9 Mon Sep 17 00:00:00 2001 From: BlockChainDev Date: Sat, 30 Mar 2019 21:34:27 +0000 Subject: [PATCH 17/24] [syncmgr] - refactor syncmgr for blockpool - add nextBlockIndex so that we can add blocks to the blockPool by comparing their index - add processBlock helper method,wraps around cfg.ProcessBlock and increments the nextBlockIndex internally --- pkg/syncmgr/syncmgr.go | 25 +++++++++++++++++++++---- 1 file changed, 21 insertions(+), 4 deletions(-) diff --git a/pkg/syncmgr/syncmgr.go b/pkg/syncmgr/syncmgr.go index bb06dfa7e..dc639a8d3 100644 --- a/pkg/syncmgr/syncmgr.go +++ b/pkg/syncmgr/syncmgr.go @@ -2,6 +2,7 @@ package syncmgr import ( "fmt" + "sync" "time" "github.com/CityOfZion/neo-go/pkg/wire/payload" @@ -50,10 +51,14 @@ type Syncmgr struct { // When receiving blocks, we can use this to determine whether the node has downloaded // all of the blocks for the last headers messages headerHash util.Uint256 + + poolLock sync.Mutex + blockPool []payload.Block + nextBlockIndex uint32 } // New creates a new sync manager -func New(cfg *Config) *Syncmgr { +func New(cfg *Config, nextBlockIndex uint32) *Syncmgr { newBlockTimer := time.AfterFunc(blockTimer, func() { cfg.AskForNewBlocks() @@ -61,9 +66,10 @@ func New(cfg *Config) *Syncmgr { newBlockTimer.Stop() return &Syncmgr{ - syncmode: headersMode, - cfg: cfg, - timer: newBlockTimer, + syncmode: headersMode, + cfg: cfg, + timer: newBlockTimer, + nextBlockIndex: nextBlockIndex, } } @@ -133,3 +139,14 @@ func (s *Syncmgr) OnBlock(peer SyncPeer, msg *payload.BlockMessage) error { func (s *Syncmgr) IsCurrent() bool { return s.syncmode == normalMode } + +func (s *Syncmgr) processBlock(block payload.Block) error { + err := s.cfg.ProcessBlock(block) + if err != nil { + return err + } + + s.nextBlockIndex++ + + return nil +} From 751d2711d478b9b8328c27e92d68b2e86269cbc1 Mon Sep 17 00:00:00 2001 From: BlockChainDev Date: Sat, 30 Mar 2019 21:36:50 +0000 Subject: [PATCH 18/24] [syncmgr] - refactored OnBlockBlockMode to use blockPool - syncmgr now checks for future blocks, instead of looking for a FutureBlockErr from the Chain. This makes it so that syncmgr does not depend on the Chain package. - removed GetBestBlockHash function from config(unused) - refactored all tests in the syncmgr package --- pkg/syncmgr/blockmode.go | 30 ++++++++++++++++----------- pkg/syncmgr/config.go | 3 --- pkg/syncmgr/mockhelpers_test.go | 4 ++-- pkg/syncmgr/normalmode.go | 2 +- pkg/syncmgr/syncmgr_onblock_test.go | 12 +++++------ pkg/syncmgr/syncmgr_onheaders_test.go | 14 ++++++------- 6 files changed, 34 insertions(+), 31 deletions(-) diff --git a/pkg/syncmgr/blockmode.go b/pkg/syncmgr/blockmode.go index 83d3acd50..406204bf5 100644 --- a/pkg/syncmgr/blockmode.go +++ b/pkg/syncmgr/blockmode.go @@ -9,22 +9,29 @@ import ( // and receives a block. func (s *Syncmgr) blockModeOnBlock(peer SyncPeer, block payload.Block) error { - // Process Block - err := s.cfg.ProcessBlock(block) - - if err == chain.ErrFutureBlock { - // XXX(Optimisation): We can cache future blocks in blockmode, if we have the corresponding header - // We can have the server cache them and sort out the semantics for when to send them to the chain - // Server can listen on chain for when a new block is saved - // or we could embed a struct in this syncmgr called blockCache, syncmgr can just tell it when it has processed - //a block and we can call ProcessBlock - return err + // Check if it is a future block + // XXX: since we are storing blocks in memory, we do not want to store blocks + // from the tip + if block.Index > s.nextBlockIndex+2000 { + return nil + } + if block.Index != s.nextBlockIndex { + s.addToBlockPool(block) + return nil } + // Process Block + err := s.processBlock(block) if err != nil && err != chain.ErrBlockAlreadyExists { return s.cfg.FetchBlockAgain(block.Hash) } + // Check the block pool + err = s.checkPool() + if err != nil { + return err + } + // Check if blockhashReceived == the header hash from last get headers this node performed // if not then increment and request next block if s.headerHash != block.Hash { @@ -32,8 +39,7 @@ func (s *Syncmgr) blockModeOnBlock(peer SyncPeer, block payload.Block) error { if err != nil { return err } - err = s.cfg.RequestBlock(nextHash, block.Index) - return err + return s.cfg.RequestBlock(nextHash, block.Index) } // If we are caught up then go into normal mode diff --git a/pkg/syncmgr/config.go b/pkg/syncmgr/config.go index 713059802..43387d414 100644 --- a/pkg/syncmgr/config.go +++ b/pkg/syncmgr/config.go @@ -24,9 +24,6 @@ type Config struct { // at the tip of this nodes chain. This assumes that the node is not in sync GetNextBlockHash func() (util.Uint256, error) - // GetBestBlockHash gets the block hash of the last saved block. - GetBestBlockHash func() (util.Uint256, error) - // AskForNewBlocks will send out a message to the network // asking for new blocks AskForNewBlocks func() diff --git a/pkg/syncmgr/mockhelpers_test.go b/pkg/syncmgr/mockhelpers_test.go index 157fdd513..d95e95e6a 100644 --- a/pkg/syncmgr/mockhelpers_test.go +++ b/pkg/syncmgr/mockhelpers_test.go @@ -89,7 +89,7 @@ func randomUint256(t *testing.T) util.Uint256 { return u } -func setupSyncMgr(mode mode) (*Syncmgr, *syncTestHelper) { +func setupSyncMgr(mode mode, nextBlockIndex uint32) (*Syncmgr, *syncTestHelper) { helper := &syncTestHelper{} cfg := &Config{ @@ -106,7 +106,7 @@ func setupSyncMgr(mode mode) (*Syncmgr, *syncTestHelper) { RequestHeaders: helper.RequestHeaders, } - syncmgr := New(cfg) + syncmgr := New(cfg, nextBlockIndex) syncmgr.syncmode = mode return syncmgr, helper diff --git a/pkg/syncmgr/normalmode.go b/pkg/syncmgr/normalmode.go index 10bbac365..ad22e52f2 100644 --- a/pkg/syncmgr/normalmode.go +++ b/pkg/syncmgr/normalmode.go @@ -43,7 +43,7 @@ func (s *Syncmgr) normalModeOnBlock(peer SyncPeer, block payload.Block) error { s.timer.Stop() // process block - err := s.cfg.ProcessBlock(block) + err := s.processBlock(block) if err != nil { s.timer.Reset(blockTimer) return s.cfg.FetchBlockAgain(block.Hash) diff --git a/pkg/syncmgr/syncmgr_onblock_test.go b/pkg/syncmgr/syncmgr_onblock_test.go index d5b79f0b3..d08aa8a74 100644 --- a/pkg/syncmgr/syncmgr_onblock_test.go +++ b/pkg/syncmgr/syncmgr_onblock_test.go @@ -11,7 +11,7 @@ import ( func TestHeadersModeOnBlock(t *testing.T) { - syncmgr, helper := setupSyncMgr(headersMode) + syncmgr, helper := setupSyncMgr(headersMode, 0) syncmgr.OnBlock(&mockPeer{}, randomBlockMessage(t, 0)) @@ -21,7 +21,7 @@ func TestHeadersModeOnBlock(t *testing.T) { func TestBlockModeOnBlock(t *testing.T) { - syncmgr, helper := setupSyncMgr(blockMode) + syncmgr, helper := setupSyncMgr(blockMode, 0) syncmgr.OnBlock(&mockPeer{}, randomBlockMessage(t, 0)) @@ -30,7 +30,7 @@ func TestBlockModeOnBlock(t *testing.T) { } func TestNormalModeOnBlock(t *testing.T) { - syncmgr, helper := setupSyncMgr(normalMode) + syncmgr, helper := setupSyncMgr(normalMode, 0) syncmgr.OnBlock(&mockPeer{}, randomBlockMessage(t, 0)) @@ -40,7 +40,7 @@ func TestNormalModeOnBlock(t *testing.T) { func TestBlockModeToNormalMode(t *testing.T) { - syncmgr, _ := setupSyncMgr(blockMode) + syncmgr, _ := setupSyncMgr(blockMode, 100) peer := &mockPeer{ height: 100, @@ -57,7 +57,7 @@ func TestBlockModeToNormalMode(t *testing.T) { } func TestBlockModeStayInBlockMode(t *testing.T) { - syncmgr, _ := setupSyncMgr(blockMode) + syncmgr, _ := setupSyncMgr(blockMode, 0) // We need our latest know hash to not be equal to the hash // of the block we received, to stay in blockmode @@ -77,7 +77,7 @@ func TestBlockModeStayInBlockMode(t *testing.T) { } func TestBlockModeAlreadyExistsErr(t *testing.T) { - syncmgr, helper := setupSyncMgr(blockMode) + syncmgr, helper := setupSyncMgr(blockMode, 100) helper.err = chain.ErrBlockAlreadyExists syncmgr.OnBlock(&mockPeer{}, randomBlockMessage(t, 100)) diff --git a/pkg/syncmgr/syncmgr_onheaders_test.go b/pkg/syncmgr/syncmgr_onheaders_test.go index 2f60a4b72..f2bcef3f3 100644 --- a/pkg/syncmgr/syncmgr_onheaders_test.go +++ b/pkg/syncmgr/syncmgr_onheaders_test.go @@ -12,7 +12,7 @@ import ( func TestHeadersModeOnHeaders(t *testing.T) { - syncmgr, helper := setupSyncMgr(headersMode) + syncmgr, helper := setupSyncMgr(headersMode, 0) syncmgr.OnHeader(&mockPeer{}, randomHeadersMessage(t, 0)) @@ -29,14 +29,14 @@ func TestHeadersModeOnHeaders(t *testing.T) { } func TestBlockModeOnHeaders(t *testing.T) { - syncmgr, helper := setupSyncMgr(blockMode) + syncmgr, helper := setupSyncMgr(blockMode, 0) // If we receive a header in blockmode, no headers will be processed syncmgr.OnHeader(&mockPeer{}, randomHeadersMessage(t, 100)) assert.Equal(t, 0, helper.headersProcessed) } func TestNormalModeOnHeadersMaxHeaders(t *testing.T) { - syncmgr, helper := setupSyncMgr(normalMode) + syncmgr, helper := setupSyncMgr(normalMode, 0) // If we receive a header in normalmode, headers will be processed syncmgr.OnHeader(&mockPeer{}, randomHeadersMessage(t, 2000)) @@ -49,7 +49,7 @@ func TestNormalModeOnHeadersMaxHeaders(t *testing.T) { // This differs from the previous function in that //we did not receive the max amount of headers func TestNormalModeOnHeaders(t *testing.T) { - syncmgr, helper := setupSyncMgr(normalMode) + syncmgr, helper := setupSyncMgr(normalMode, 0) // If we receive a header in normalmode, headers will be processed syncmgr.OnHeader(&mockPeer{}, randomHeadersMessage(t, 200)) @@ -60,7 +60,7 @@ func TestNormalModeOnHeaders(t *testing.T) { } func TestLastHeaderUpdates(t *testing.T) { - syncmgr, _ := setupSyncMgr(headersMode) + syncmgr, _ := setupSyncMgr(headersMode, 0) hdrsMessage := randomHeadersMessage(t, 200) hdrs := hdrsMessage.Headers @@ -95,7 +95,7 @@ func TestLastHeaderUpdates(t *testing.T) { func TestHeadersModeOnHeadersErr(t *testing.T) { - syncmgr, helper := setupSyncMgr(headersMode) + syncmgr, helper := setupSyncMgr(headersMode, 0) helper.err = &chain.ValidationError{} syncmgr.OnHeader(&mockPeer{}, randomHeadersMessage(t, 200)) @@ -106,7 +106,7 @@ func TestHeadersModeOnHeadersErr(t *testing.T) { } func TestNormalModeOnHeadersErr(t *testing.T) { - syncmgr, helper := setupSyncMgr(normalMode) + syncmgr, helper := setupSyncMgr(normalMode, 0) helper.err = &chain.ValidationError{} syncmgr.OnHeader(&mockPeer{}, randomHeadersMessage(t, 200)) From 68b0e2e3f2d545c3b38328250e3d59af7c1100d0 Mon Sep 17 00:00:00 2001 From: BlockChainDev Date: Sat, 30 Mar 2019 21:39:39 +0000 Subject: [PATCH 19/24] [server] - modified server to pass the syncmgr the lastBlock.Index+1 so that syncmgr knows what blockIndex it should be looking for upon initialisation --- pkg/server/server.go | 5 ++++- pkg/server/syncmgr.go | 12 ++++++++++-- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/pkg/server/server.go b/pkg/server/server.go index b1e4c0f5b..7730ee79b 100644 --- a/pkg/server/server.go +++ b/pkg/server/server.go @@ -56,7 +56,10 @@ func New(net protocol.Magic, port uint16) (*Server, error) { s.chain = chain // Setup sync manager - syncmgr := setupSyncManager(s) + syncmgr, err := setupSyncManager(s) + if err != nil { + return nil, err + } s.smg = syncmgr // Setup connection manager diff --git a/pkg/server/syncmgr.go b/pkg/server/syncmgr.go index 7de87188b..e6020d5b9 100644 --- a/pkg/server/syncmgr.go +++ b/pkg/server/syncmgr.go @@ -11,7 +11,7 @@ import ( "github.com/CityOfZion/neo-go/pkg/wire/util" ) -func setupSyncManager(s *Server) *syncmgr.Syncmgr { +func setupSyncManager(s *Server) (*syncmgr.Syncmgr, error) { cfg := &syncmgr.Config{ ProcessBlock: s.processBlock, @@ -27,7 +27,15 @@ func setupSyncManager(s *Server) *syncmgr.Syncmgr { FetchBlockAgain: s.fetchBlockAgain, } - return syncmgr.New(cfg) + // Add nextBlockIndex in syncmgr + lastBlock, err := s.chain.Db.GetLastBlock() + if err != nil { + return nil, err + } + + nextBlockIndex := lastBlock.Index + 1 + + return syncmgr.New(cfg, nextBlockIndex), nil } func (s *Server) onHeader(peer *peer.Peer, hdrsMessage *payload.HeadersMessage) { From 06e71d9b7e166025201b8120e10a883c2d2db379 Mon Sep 17 00:00:00 2001 From: BlockChainDev Date: Sat, 30 Mar 2019 21:44:24 +0000 Subject: [PATCH 20/24] [syncmgr] - update comment --- pkg/syncmgr/blockpool.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/syncmgr/blockpool.go b/pkg/syncmgr/blockpool.go index b038b29f3..2b1f37761 100644 --- a/pkg/syncmgr/blockpool.go +++ b/pkg/syncmgr/blockpool.go @@ -44,7 +44,7 @@ func (s *Syncmgr) checkPool() error { break } - // Save this block and save the indice so we can remove it + // Save this block and save the indice location so we can remove it, when we defer err := s.processBlock(block) if err != nil { return err From 336fc02ad5b7dba862e20b1dcc9189214cb38ad5 Mon Sep 17 00:00:00 2001 From: BlockChainDev Date: Sat, 30 Mar 2019 22:42:07 +0000 Subject: [PATCH 21/24] [syncmgr] - Fix bug; accounts for a fresh database startup and we only have the genesis block --- pkg/syncmgr/blockmode.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/syncmgr/blockmode.go b/pkg/syncmgr/blockmode.go index 406204bf5..f357ca642 100644 --- a/pkg/syncmgr/blockmode.go +++ b/pkg/syncmgr/blockmode.go @@ -15,7 +15,7 @@ func (s *Syncmgr) blockModeOnBlock(peer SyncPeer, block payload.Block) error { if block.Index > s.nextBlockIndex+2000 { return nil } - if block.Index != s.nextBlockIndex { + if block.Index > s.nextBlockIndex { s.addToBlockPool(block) return nil } From fb672c00adc58279ebb7a77f111eb398bc00ecdb Mon Sep 17 00:00:00 2001 From: decentralisedkev <37423678+decentralisedkev@users.noreply.github.com> Date: Sat, 30 Mar 2019 23:52:28 +0000 Subject: [PATCH 22/24] [travis] add verbose flag to test --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index cf7f94759..630f4078d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -9,9 +9,9 @@ install: - go mod tidy -v script: - golint -set_exit_status ./... - - go test -race -coverprofile=coverage.txt -covermode=atomic ./... + - go test -v -race -coverprofile=coverage.txt -covermode=atomic ./... after_success: - bash <(curl -s https://codecov.io/bash) matrix: allow_failures: - - go: tip \ No newline at end of file + - go: tip From 12149c93790fb2a7b613b2de54f4ce59f247c1f5 Mon Sep 17 00:00:00 2001 From: BlockChainDev Date: Sun, 31 Mar 2019 00:24:10 +0000 Subject: [PATCH 23/24] [server] - remove extra call after adding a peer - switched endian for headers endian switch --- pkg/server/connmgr.go | 15 --------------- pkg/server/server.go | 3 ++- 2 files changed, 2 insertions(+), 16 deletions(-) diff --git a/pkg/server/connmgr.go b/pkg/server/connmgr.go index 64b16bdaf..e97416801 100644 --- a/pkg/server/connmgr.go +++ b/pkg/server/connmgr.go @@ -1,7 +1,6 @@ package server import ( - "encoding/hex" "fmt" "net" "strconv" @@ -9,7 +8,6 @@ import ( "github.com/CityOfZion/neo-go/pkg/connmgr" "github.com/CityOfZion/neo-go/pkg/peer" - "github.com/CityOfZion/neo-go/pkg/wire/util" iputils "github.com/CityOfZion/neo-go/pkg/wire/util/ip" ) @@ -34,19 +32,6 @@ func (s *Server) onConnection(conn net.Conn, addr string) { } s.pmg.AddPeer(p) - - byt, err := hex.DecodeString("d42561e3d30e15be6400b6df2f328e02d2bf6354c41dce433bc57687c82144bf") - if err != nil { - fmt.Println("Error getting hash " + err.Error()) - } - lh, err := util.Uint256DecodeBytes(byt) - if err != nil { - fmt.Println("Error getting hash " + err.Error()) - } - err = p.RequestHeaders(lh.Reverse()) - if err != nil { - fmt.Println(err) - } } func (s *Server) onAccept(conn net.Conn) { diff --git a/pkg/server/server.go b/pkg/server/server.go index 7730ee79b..22a781170 100644 --- a/pkg/server/server.go +++ b/pkg/server/server.go @@ -95,7 +95,8 @@ func (s *Server) Run() error { if err != nil { return err } - err = s.pmg.RequestHeaders(bestHeader.Hash.Reverse()) + + err = s.pmg.RequestHeaders(bestHeader.Hash) if err != nil { return err } From b9b118d3ea8eb9c772f1bf9f37b52d0f9777f573 Mon Sep 17 00:00:00 2001 From: dauTT <30392990+dauTT@users.noreply.github.com> Date: Mon, 5 Aug 2019 09:34:31 +0200 Subject: [PATCH 24/24] Add ReverseString, ToReverseScriptHash method (#281) * Added 1) ReverseString method to the Uint160 type 2) ToReverseScriptHash method to convert a base58 address to a reverse script hash * Simplified ToScriptHash method --- pkg/wire/util/address/address.go | 19 ++++++++++++------- pkg/wire/util/address/address_test.go | 17 +++++++++++++++++ pkg/wire/util/uint160.go | 5 +++++ pkg/wire/util/uint160_test.go | 12 ++++++++++++ 4 files changed, 46 insertions(+), 7 deletions(-) create mode 100644 pkg/wire/util/address/address_test.go diff --git a/pkg/wire/util/address/address.go b/pkg/wire/util/address/address.go index 12f09c63a..08b0347af 100644 --- a/pkg/wire/util/address/address.go +++ b/pkg/wire/util/address/address.go @@ -1,22 +1,27 @@ package address import ( - "encoding/hex" - "github.com/CityOfZion/neo-go/pkg/crypto/base58" "github.com/CityOfZion/neo-go/pkg/wire/util" ) // ToScriptHash converts an address to a script hash func ToScriptHash(address string) string { - - decodedAddressAsBytes, err := base58.Decode(address) + a, err := Uint160Decode(address) if err != nil { return "" } - decodedAddressAsHex := hex.EncodeToString(decodedAddressAsBytes) - scriptHash := (decodedAddressAsHex[2:42]) - return scriptHash + return a.String() + +} + +// ToReverseScriptHash converts an address to a reverse script hash +func ToReverseScriptHash(address string) string { + a, err := Uint160Decode(address) + if err != nil { + return "" + } + return a.ReverseString() } // FromUint160 returns the "NEO address" from the given diff --git a/pkg/wire/util/address/address_test.go b/pkg/wire/util/address/address_test.go new file mode 100644 index 000000000..af264a935 --- /dev/null +++ b/pkg/wire/util/address/address_test.go @@ -0,0 +1,17 @@ +package address + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestScriptHash(t *testing.T) { + address := "AJeAEsmeD6t279Dx4n2HWdUvUmmXQ4iJvP" + + hash := ToScriptHash(address) + reverseHash := ToReverseScriptHash(address) + + assert.Equal(t, "b28427088a3729b2536d10122960394e8be6721f", reverseHash) + assert.Equal(t, "1f72e68b4e39602912106d53b229378a082784b2", hash) +} diff --git a/pkg/wire/util/uint160.go b/pkg/wire/util/uint160.go index 429fe5d95..fa81299bf 100644 --- a/pkg/wire/util/uint160.go +++ b/pkg/wire/util/uint160.go @@ -68,6 +68,11 @@ func (u Uint160) String() string { return hex.EncodeToString(u.Bytes()) } +// ReverseString implements the stringer interface. +func (u Uint160) ReverseString() string { + return hex.EncodeToString(u.BytesReverse()) +} + // Equals returns true if both Uint256 values are the same. func (u Uint160) Equals(other Uint160) bool { for i := 0; i < uint160Size; i++ { diff --git a/pkg/wire/util/uint160_test.go b/pkg/wire/util/uint160_test.go index 96416653c..f92f736c7 100644 --- a/pkg/wire/util/uint160_test.go +++ b/pkg/wire/util/uint160_test.go @@ -48,3 +48,15 @@ func TestUInt160Equals(t *testing.T) { t.Fatalf("%s and %s must be equal", ua, ua) } } + +func TestUInt160String(t *testing.T) { + hexStr := "b28427088a3729b2536d10122960394e8be6721f" + hexRevStr := "1f72e68b4e39602912106d53b229378a082784b2" + + val, err := Uint160DecodeString(hexStr) + assert.Nil(t, err) + + assert.Equal(t, hexStr, val.String()) + assert.Equal(t, hexRevStr, val.ReverseString()) + +}