Merge pull request #472 from nspcc-dev/feature/serialize

VM: Implement serialization and deserealization interops

This PR implements Serialize/Deserialize interops.

Closes #419.

pkg/core/interop_neo.go

@@ -736,3 +736,13 @@ func (ic *interopContext) assetRenew(v *vm.VM) error {
 	v.Estack().PushVal(expiration)
 	return nil
 }
+
+// runtimeSerialize serializes top stack item into a ByteArray.
+func (ic *interopContext) runtimeSerialize(v *vm.VM) error {
+	return vm.RuntimeSerialize(v)
+}
+
+// runtimeDeserialize deserializes ByteArray from a stack into an item.
+func (ic *interopContext) runtimeDeserialize(v *vm.VM) error {
+	return vm.RuntimeDeserialize(v)
+}

pkg/core/interops.go

@@ -64,8 +64,8 @@ func (ic *interopContext) getSystemInteropMap() map[string]vm.InteropFuncPrice {
 		"System.Storage.PutEx":                          {Func: ic.storagePutEx, Price: 0},
 		"System.StorageContext.AsReadOnly":              {Func: ic.storageContextAsReadOnly, Price: 1},
 		"System.Transaction.GetHash":                    {Func: ic.txGetHash, Price: 1},
-		//		"System.Runtime.Deserialize":                    {Func: ic.runtimeDeserialize, Price: 1},
-		//		"System.Runtime.Serialize":                      {Func: ic.runtimeSerialize, Price: 1},
+		"System.Runtime.Deserialize":                    {Func: ic.runtimeDeserialize, Price: 1},
+		"System.Runtime.Serialize":                      {Func: ic.runtimeSerialize, Price: 1},
 	}
 }
 
@@ -148,8 +148,8 @@ func (ic *interopContext) getNeoInteropMap() map[string]vm.InteropFuncPrice {
 		//		"Neo.Iterator.Key": {Func: ic.iteratorKey, Price: 1},
 		//		"Neo.Iterator.Keys": {Func: ic.iteratorKeys, Price: 1},
 		//		"Neo.Iterator.Values": {Func: ic.iteratorValues, Price: 1},
-		//		"Neo.Runtime.Deserialize":  {Func: ic.runtimeDeserialize, Price: 1},
-		//		"Neo.Runtime.Serialize":    {Func: ic.runtimeSerialize, Price: 1},
+		"Neo.Runtime.Deserialize": {Func: ic.runtimeDeserialize, Price: 1},
+		"Neo.Runtime.Serialize":   {Func: ic.runtimeSerialize, Price: 1},
 		//		"Neo.Storage.Find":                {Func: ic.storageFind, Price: 1},
 		//		"Neo.Witness.GetVerificationScript": {Func: ic.witnessGetVerificationScript, Price: 100},
 

pkg/vm/interop.go

@@ -1,6 +1,7 @@
 package vm
 
 import (
+	"errors"
 	"fmt"
 )
 
@@ -20,3 +21,32 @@ func runtimeNotify(vm *VM) error {
 	fmt.Printf("NEO-GO-VM (notify) > %s\n", item.Value())
 	return nil
 }
+
+// RuntimeSerialize handles syscalls System.Runtime.Serialize and Neo.Runtime.Serialize.
+func RuntimeSerialize(vm *VM) error {
+	item := vm.Estack().Pop()
+	data, err := serializeItem(item.value)
+	if err != nil {
+		return err
+	} else if len(data) > MaxItemSize {
+		return errors.New("too big item")
+	}
+
+	vm.Estack().PushVal(data)
+
+	return nil
+}
+
+// RuntimeDeserialize handles syscalls System.Runtime.Deserialize and Neo.Runtime.Deserialize.
+func RuntimeDeserialize(vm *VM) error {
+	data := vm.Estack().Pop().Bytes()
+
+	item, err := deserializeItem(data)
+	if err != nil {
+		return err
+	}
+
+	vm.Estack().Push(&Element{value: item})
+
+	return nil
+}

pkg/vm/serialization.go

@@ -0,0 +1,130 @@
+package vm
+
+import (
+	"errors"
+	"math/big"
+
+	"github.com/CityOfZion/neo-go/pkg/io"
+	"github.com/CityOfZion/neo-go/pkg/util"
+)
+
+type stackItemType byte
+
+const (
+	byteArrayT stackItemType = 0x00
+	booleanT   stackItemType = 0x01
+	integerT   stackItemType = 0x02
+	arrayT     stackItemType = 0x80
+	structT    stackItemType = 0x81
+	mapT       stackItemType = 0x82
+)
+
+func serializeItem(item StackItem) ([]byte, error) {
+	w := io.NewBufBinWriter()
+	serializeItemTo(item, w.BinWriter, make(map[StackItem]bool))
+	if w.Err != nil {
+		return nil, w.Err
+	}
+	return w.Bytes(), nil
+}
+
+func serializeItemTo(item StackItem, w *io.BinWriter, seen map[StackItem]bool) {
+	if seen[item] {
+		w.Err = errors.New("recursive structures are not supported")
+		return
+	}
+	seen[item] = true
+
+	switch t := item.(type) {
+	case *ByteArrayItem:
+		w.WriteLE(byte(byteArrayT))
+		w.WriteBytes(t.value)
+	case *BoolItem:
+		w.WriteLE(byte(booleanT))
+		w.WriteLE(t.value)
+	case *BigIntegerItem:
+		w.WriteLE(byte(integerT))
+		w.WriteBytes(t.Bytes())
+	case *InteropItem:
+		w.Err = errors.New("not supported")
+	case *ArrayItem, *StructItem:
+		_, isArray := t.(*ArrayItem)
+		if isArray {
+			w.WriteLE(byte(arrayT))
+		} else {
+			w.WriteLE(byte(structT))
+		}
+
+		arr := t.Value().([]StackItem)
+		w.WriteVarUint(uint64(len(arr)))
+		for i := range arr {
+			serializeItemTo(arr[i], w, seen)
+		}
+	case *MapItem:
+		w.WriteLE(byte(mapT))
+		w.WriteVarUint(uint64(len(t.value)))
+		for k, v := range t.value {
+			serializeItemTo(v, w, seen)
+			serializeItemTo(makeStackItem(k), w, seen)
+		}
+	}
+}
+
+func deserializeItem(data []byte) (StackItem, error) {
+	r := io.NewBinReaderFromBuf(data)
+	item := deserializeItemFrom(r)
+	if r.Err != nil {
+		return nil, r.Err
+	}
+	return item, nil
+}
+
+func deserializeItemFrom(r *io.BinReader) StackItem {
+	var t byte
+	r.ReadLE(&t)
+	if r.Err != nil {
+		return nil
+	}
+
+	switch stackItemType(t) {
+	case byteArrayT:
+		data := r.ReadBytes()
+		return NewByteArrayItem(data)
+	case booleanT:
+		var b bool
+		r.ReadLE(&b)
+		return NewBoolItem(b)
+	case integerT:
+		data := r.ReadBytes()
+		num := new(big.Int).SetBytes(util.ArrayReverse(data))
+		return &BigIntegerItem{
+			value: num,
+		}
+	case arrayT, structT:
+		size := int(r.ReadVarUint())
+		arr := make([]StackItem, size)
+		for i := 0; i < size; i++ {
+			arr[i] = deserializeItemFrom(r)
+		}
+
+		if stackItemType(t) == arrayT {
+			return &ArrayItem{value: arr}
+		}
+		return &StructItem{value: arr}
+	case mapT:
+		size := int(r.ReadVarUint())
+		m := NewMapItem()
+		for i := 0; i < size; i++ {
+			value := deserializeItemFrom(r)
+			key := deserializeItemFrom(r)
+			if r.Err != nil {
+				break
+			}
+			m.Add(key, value)
+		}
+		return m
+	default:
+		r.Err = errors.New("unknown type")
+		return nil
+	}
+}

pkg/vm/stack.go

@@ -122,7 +122,7 @@ func (e *Element) Bytes() []byte {
 	case *ByteArrayItem:
 		return t.value
 	case *BigIntegerItem:
-		return util.ArrayReverse(t.value.Bytes()) // neoVM returns in LE
+		return t.Bytes() // neoVM returns in LE
 	case *BoolItem:
 		if t.value {
 			return []byte{1}

pkg/vm/stack_item.go

@@ -6,6 +6,8 @@ import (
 	"fmt"
 	"math/big"
 	"reflect"
+
+	"github.com/CityOfZion/neo-go/pkg/util"
 )
 
 // A StackItem represents the "real" value that is pushed on the stack.
@@ -131,6 +133,11 @@ func NewBigIntegerItem(value int) *BigIntegerItem {
 	}
 }
 
+// Bytes converts i to a slice of bytes.
+func (i *BigIntegerItem) Bytes() []byte {
+	return util.ArrayReverse(i.value.Bytes())
+}
+
 // Value implements StackItem interface.
 func (i *BigIntegerItem) Value() interface{} {
 	return i.value

pkg/vm/vm.go

@@ -86,6 +86,10 @@ func New() *VM {
 	// Register native interop hooks.
 	vm.RegisterInteropFunc("Neo.Runtime.Log", runtimeLog, 1)
 	vm.RegisterInteropFunc("Neo.Runtime.Notify", runtimeNotify, 1)
+	vm.RegisterInteropFunc("Neo.Runtime.Serialize", RuntimeSerialize, 1)
+	vm.RegisterInteropFunc("System.Runtime.Serialize", RuntimeSerialize, 1)
+	vm.RegisterInteropFunc("Neo.Runtime.Deserialize", RuntimeDeserialize, 1)
+	vm.RegisterInteropFunc("System.Runtime.Deserialize", RuntimeDeserialize, 1)
 
 	return vm
 }

pkg/vm/vm_test.go

@@ -197,6 +197,147 @@ func TestPushData4Good(t *testing.T) {
 	assert.Equal(t, []byte{1, 2, 3}, vm.estack.Pop().Bytes())
 }
 
+func getSyscallProg(name string) (prog []byte) {
+	prog = []byte{byte(SYSCALL)}
+	prog = append(prog, byte(len(name)))
+	prog = append(prog, name...)
+
+	return
+}
+
+func getSerializeProg() (prog []byte) {
+	prog = append(prog, getSyscallProg("Neo.Runtime.Serialize")...)
+	prog = append(prog, getSyscallProg("Neo.Runtime.Deserialize")...)
+	prog = append(prog, byte(RET))
+
+	return
+}
+
+func testSerialize(t *testing.T, vm *VM) {
+	err := vm.Step()
+	require.NoError(t, err)
+	require.Equal(t, 1, vm.estack.Len())
+	require.IsType(t, (*ByteArrayItem)(nil), vm.estack.Top().value)
+
+	err = vm.Step()
+	require.NoError(t, err)
+	require.Equal(t, 1, vm.estack.Len())
+}
+
+func TestSerializeBool(t *testing.T) {
+	vm := load(getSerializeProg())
+	vm.estack.PushVal(true)
+
+	testSerialize(t, vm)
+
+	require.IsType(t, (*BoolItem)(nil), vm.estack.Top().value)
+	require.Equal(t, true, vm.estack.Top().Bool())
+}
+
+func TestSerializeByteArray(t *testing.T) {
+	vm := load(getSerializeProg())
+	value := []byte{1, 2, 3}
+	vm.estack.PushVal(value)
+
+	testSerialize(t, vm)
+
+	require.IsType(t, (*ByteArrayItem)(nil), vm.estack.Top().value)
+	require.Equal(t, value, vm.estack.Top().Bytes())
+}
+
+func TestSerializeInteger(t *testing.T) {
+	vm := load(getSerializeProg())
+	value := int64(123)
+	vm.estack.PushVal(value)
+
+	testSerialize(t, vm)
+
+	require.IsType(t, (*BigIntegerItem)(nil), vm.estack.Top().value)
+	require.Equal(t, value, vm.estack.Top().BigInt().Int64())
+}
+
+func TestSerializeArray(t *testing.T) {
+	vm := load(getSerializeProg())
+	item := NewArrayItem([]StackItem{
+		makeStackItem(true),
+		makeStackItem(123),
+		NewMapItem(),
+	})
+
+	vm.estack.Push(&Element{value: item})
+
+	testSerialize(t, vm)
+
+	require.IsType(t, (*ArrayItem)(nil), vm.estack.Top().value)
+	require.Equal(t, item.value, vm.estack.Top().Array())
+}
+
+func TestSerializeArrayBad(t *testing.T) {
+	vm := load(getSerializeProg())
+	item := NewArrayItem(makeArrayOfFalses(2))
+	item.value[1] = item
+
+	vm.estack.Push(&Element{value: item})
+
+	err := vm.Step()
+	require.Error(t, err)
+	require.True(t, vm.HasFailed())
+}
+
+func TestSerializeStruct(t *testing.T) {
+	vm := load(getSerializeProg())
+	item := NewStructItem([]StackItem{
+		makeStackItem(true),
+		makeStackItem(123),
+		NewMapItem(),
+	})
+
+	vm.estack.Push(&Element{value: item})
+
+	testSerialize(t, vm)
+
+	require.IsType(t, (*StructItem)(nil), vm.estack.Top().value)
+	require.Equal(t, item.value, vm.estack.Top().Array())
+}
+
+func TestDeserializeUnknown(t *testing.T) {
+	prog := append(getSyscallProg("Neo.Runtime.Deserialize"), byte(RET))
+	vm := load(prog)
+
+	data, err := serializeItem(NewBigIntegerItem(123))
+	require.NoError(t, err)
+
+	data[0] = 0xFF
+	vm.estack.PushVal(data)
+
+	checkVMFailed(t, vm)
+}
+
+func TestSerializeMap(t *testing.T) {
+	vm := load(getSerializeProg())
+	item := NewMapItem()
+	item.Add(makeStackItem(true), makeStackItem([]byte{1, 2, 3}))
+	item.Add(makeStackItem([]byte{0}), makeStackItem(false))
+
+	vm.estack.Push(&Element{value: item})
+
+	testSerialize(t, vm)
+
+	require.IsType(t, (*MapItem)(nil), vm.estack.Top().value)
+	require.Equal(t, item.value, vm.estack.Top().value.(*MapItem).value)
+}
+
+func TestSerializeInterop(t *testing.T) {
+	vm := load(getSerializeProg())
+	item := NewInteropItem("kek")
+
+	vm.estack.Push(&Element{value: item})
+
+	err := vm.Step()
+	require.Error(t, err)
+	require.True(t, vm.HasFailed())
+}
+
 func callNTimes(n uint16) []byte {
 	return makeProgram(
 		PUSHBYTES2, Instruction(n), Instruction(n>>8), // little-endian