Merge pull request #2252 from nspcc-dev/packmap-packstruct

vm: add PACKMAP/PACKSTRUCT, extend UNPACK

cli/nep17_test.go

@@ -85,7 +85,7 @@ func TestNEP17Balance(t *testing.T) {
 		}
 
 		e.checkNextLine(t, "^\\s*$")
-		addr4, err := address.StringToUint160("NU4CTk9H2fgNCuC3ZPqX4LjUX3MHt3Rh6p") // deployed verify.go contract
+		addr4, err := address.StringToUint160("NQ3nAdFQXzemHC9uvr4af2Ysap6aZJpqgN") // deployed verify.go contract
 		require.NoError(t, err)
 		e.checkNextLine(t, "^Account "+address.Uint160ToString(addr4))
 		e.checkEOF(t)

cli/testdata/wallet1_solo.json

@@ -61,7 +61,7 @@
       "isDefault": false
     },
     {
-      "address": "NU4CTk9H2fgNCuC3ZPqX4LjUX3MHt3Rh6p",
+      "address": "NQ3nAdFQXzemHC9uvr4af2Ysap6aZJpqgN",
       "key": "6PYSATFztBa3CHjSR6sLAKungUEAbQUCVE16KzmaQQ38gLeYGZ9Knd5mGv",
       "label": "verify",
       "contract": {

pkg/compiler/codegen.go

@@ -324,14 +324,11 @@ func (c *codegen) emitDefault(t types.Type) {
 		}
 	case *types.Struct:
 		num := t.NumFields()
-		emit.Int(c.prog.BinWriter, int64(num))
+		for i := num - 1; i >= 0; i-- {
-		emit.Opcodes(c.prog.BinWriter, opcode.NEWSTRUCT)
-		for i := 0; i < num; i++ {
-			emit.Opcodes(c.prog.BinWriter, opcode.DUP)
-			emit.Int(c.prog.BinWriter, int64(i))
 			c.emitDefault(t.Field(i).Type())
-			emit.Opcodes(c.prog.BinWriter, opcode.SETITEM)
 		}
+		emit.Int(c.prog.BinWriter, int64(num))
+		emit.Opcodes(c.prog.BinWriter, opcode.PACKSTRUCT)
 	default:
 		emit.Opcodes(c.prog.BinWriter, opcode.PUSHNULL)
 	}
@@ -1574,10 +1571,6 @@ func (c *codegen) convertSyscall(f *funcScope, expr *ast.CallExpr) {
 	if strings.HasPrefix(f.name, "Syscall") {
 		c.emitReverse(len(expr.Args) - 1)
 		emit.Syscall(c.prog.BinWriter, name)
-
-		// This NOP instruction is basically not needed, but if we do, we have a
-		// one to one matching avm file with neo-python which is very nice for debugging.
-		emit.Opcodes(c.prog.BinWriter, opcode.NOP)
 	} else {
 		op, err := opcode.FromString(name)
 		if err != nil {
@@ -1796,14 +1789,14 @@ func (c *codegen) convertByteArray(elems []ast.Expr) {
 }
 
 func (c *codegen) convertMap(lit *ast.CompositeLit) {
-	emit.Opcodes(c.prog.BinWriter, opcode.NEWMAP)
+	l := len(lit.Elts)
-	for i := range lit.Elts {
+	for i := l - 1; i >= 0; i-- {
 		elem := lit.Elts[i].(*ast.KeyValueExpr)
-		emit.Opcodes(c.prog.BinWriter, opcode.DUP)
-		ast.Walk(c, elem.Key)
 		ast.Walk(c, elem.Value)
-		emit.Opcodes(c.prog.BinWriter, opcode.SETITEM)
+		ast.Walk(c, elem.Key)
 	}
+	emit.Int(c.prog.BinWriter, int64(l))
+	emit.Opcodes(c.prog.BinWriter, opcode.PACKMAP)
 }
 
 func (c *codegen) getStruct(typ types.Type) (*types.Struct, bool) {
@@ -1827,14 +1820,6 @@ func (c *codegen) convertStruct(lit *ast.CompositeLit, ptr bool) {
 		return
 	}
 
-	emit.Opcodes(c.prog.BinWriter, opcode.NOP)
-	emit.Int(c.prog.BinWriter, int64(strct.NumFields()))
-	if ptr {
-		emit.Opcodes(c.prog.BinWriter, opcode.NEWARRAY)
-	} else {
-		emit.Opcodes(c.prog.BinWriter, opcode.NEWSTRUCT)
-	}
-
 	keyedLit := len(lit.Elts) > 0
 	if keyedLit {
 		_, ok := lit.Elts[0].(*ast.KeyValueExpr)
@@ -1842,13 +1827,10 @@ func (c *codegen) convertStruct(lit *ast.CompositeLit, ptr bool) {
 	}
 	// We need to locally store all the fields, even if they are not initialized.
 	// We will initialize all fields to their "zero" value.
-	for i := 0; i < strct.NumFields(); i++ {
+	for i := strct.NumFields() - 1; i >= 0; i-- {
 		sField := strct.Field(i)
 		var initialized bool
 
-		emit.Opcodes(c.prog.BinWriter, opcode.DUP)
-		emit.Int(c.prog.BinWriter, int64(i))
-
 		if !keyedLit {
 			if len(lit.Elts) > i {
 				ast.Walk(c, lit.Elts[i])
@@ -1870,7 +1852,12 @@ func (c *codegen) convertStruct(lit *ast.CompositeLit, ptr bool) {
 		if !initialized {
 			c.emitDefault(sField.Type())
 		}
-		emit.Opcodes(c.prog.BinWriter, opcode.SETITEM)
+	}
+	emit.Int(c.prog.BinWriter, int64(strct.NumFields()))
+	if ptr {
+		emit.Opcodes(c.prog.BinWriter, opcode.PACK)
+	} else {
+		emit.Opcodes(c.prog.BinWriter, opcode.PACKSTRUCT)
 	}
 }
 

pkg/compiler/struct_test.go

@@ -394,6 +394,16 @@ var structTestCases = []testCase{
 		}`,
 		big.NewInt(11),
 	},
+	{
+		"lengthy struct default value",
+		`package foo
+		type S struct { x int; y []byte; z bool }
+		func Main() int {
+			var s S
+			return s.x
+		}`,
+		big.NewInt(0),
+	},
 	{
 		"nested selectors (complex write)",
 		`package foo

pkg/core/fee/opcode.go

@@ -180,6 +180,8 @@ var coefficients = [256]uint16{
 	opcode.MIN:          1 << 3,
 	opcode.MAX:          1 << 3,
 	opcode.WITHIN:       1 << 3,
+	opcode.PACKMAP:      1 << 11,
+	opcode.PACKSTRUCT:   1 << 11,
 	opcode.PACK:         1 << 11,
 	opcode.UNPACK:       1 << 11,
 	opcode.NEWARRAY0:    1 << 4,

pkg/vm/opcode/opcode.go

@@ -196,6 +196,8 @@ const (
 	WITHIN      Opcode = 0xBB
 
 	// Advanced data structures (arrays, structures, maps).
+	PACKMAP      Opcode = 0xBE
+	PACKSTRUCT   Opcode = 0xBF
 	PACK         Opcode = 0xC0
 	UNPACK       Opcode = 0xC1
 	NEWARRAY0    Opcode = 0xC2

pkg/vm/opcode/opcode_string.go

@@ -176,6 +176,8 @@ func _() {
 	_ = x[MIN-185]
 	_ = x[MAX-186]
 	_ = x[WITHIN-187]
+	_ = x[PACKMAP-190]
+	_ = x[PACKSTRUCT-191]
 	_ = x[PACK-192]
 	_ = x[UNPACK-193]
 	_ = x[NEWARRAY0-194]
@@ -200,7 +202,7 @@ func _() {
 	_ = x[CONVERT-219]
 }
 
-const _Opcode_name = "PUSHINT8PUSHINT16PUSHINT32PUSHINT64PUSHINT128PUSHINT256PUSHAPUSHNULLPUSHDATA1PUSHDATA2PUSHDATA4PUSHM1PUSH0PUSH1PUSH2PUSH3PUSH4PUSH5PUSH6PUSH7PUSH8PUSH9PUSH10PUSH11PUSH12PUSH13PUSH14PUSH15PUSH16NOPJMPJMP_LJMPIFJMPIF_LJMPIFNOTJMPIFNOT_LJMPEQJMPEQ_LJMPNEJMPNE_LJMPGTJMPGT_LJMPGEJMPGE_LJMPLTJMPLT_LJMPLEJMPLE_LCALLCALL_LCALLACALLTABORTASSERTTHROWTRYTRY_LENDTRYENDTRY_LENDFINALLYRETSYSCALLDEPTHDROPNIPXDROPCLEARDUPOVERPICKTUCKSWAPROTROLLREVERSE3REVERSE4REVERSENINITSSLOTINITSLOTLDSFLD0LDSFLD1LDSFLD2LDSFLD3LDSFLD4LDSFLD5LDSFLD6LDSFLDSTSFLD0STSFLD1STSFLD2STSFLD3STSFLD4STSFLD5STSFLD6STSFLDLDLOC0LDLOC1LDLOC2LDLOC3LDLOC4LDLOC5LDLOC6LDLOCSTLOC0STLOC1STLOC2STLOC3STLOC4STLOC5STLOC6STLOCLDARG0LDARG1LDARG2LDARG3LDARG4LDARG5LDARG6LDARGSTARG0STARG1STARG2STARG3STARG4STARG5STARG6STARGNEWBUFFERMEMCPYCATSUBSTRLEFTRIGHTINVERTANDORXOREQUALNOTEQUALSIGNABSNEGATEINCDECADDSUBMULDIVMODPOWSQRTSHLSHRNOTBOOLANDBOOLORNZNUMEQUALNUMNOTEQUALLTLEGTGEMINMAXWITHINPACKUNPACKNEWARRAY0NEWARRAYNEWARRAY_TNEWSTRUCT0NEWSTRUCTNEWMAPSIZEHASKEYKEYSVALUESPICKITEMAPPENDSETITEMREVERSEITEMSREMOVECLEARITEMSPOPITEMISNULLISTYPECONVERT"
+const _Opcode_name = "PUSHINT8PUSHINT16PUSHINT32PUSHINT64PUSHINT128PUSHINT256PUSHAPUSHNULLPUSHDATA1PUSHDATA2PUSHDATA4PUSHM1PUSH0PUSH1PUSH2PUSH3PUSH4PUSH5PUSH6PUSH7PUSH8PUSH9PUSH10PUSH11PUSH12PUSH13PUSH14PUSH15PUSH16NOPJMPJMP_LJMPIFJMPIF_LJMPIFNOTJMPIFNOT_LJMPEQJMPEQ_LJMPNEJMPNE_LJMPGTJMPGT_LJMPGEJMPGE_LJMPLTJMPLT_LJMPLEJMPLE_LCALLCALL_LCALLACALLTABORTASSERTTHROWTRYTRY_LENDTRYENDTRY_LENDFINALLYRETSYSCALLDEPTHDROPNIPXDROPCLEARDUPOVERPICKTUCKSWAPROTROLLREVERSE3REVERSE4REVERSENINITSSLOTINITSLOTLDSFLD0LDSFLD1LDSFLD2LDSFLD3LDSFLD4LDSFLD5LDSFLD6LDSFLDSTSFLD0STSFLD1STSFLD2STSFLD3STSFLD4STSFLD5STSFLD6STSFLDLDLOC0LDLOC1LDLOC2LDLOC3LDLOC4LDLOC5LDLOC6LDLOCSTLOC0STLOC1STLOC2STLOC3STLOC4STLOC5STLOC6STLOCLDARG0LDARG1LDARG2LDARG3LDARG4LDARG5LDARG6LDARGSTARG0STARG1STARG2STARG3STARG4STARG5STARG6STARGNEWBUFFERMEMCPYCATSUBSTRLEFTRIGHTINVERTANDORXOREQUALNOTEQUALSIGNABSNEGATEINCDECADDSUBMULDIVMODPOWSQRTSHLSHRNOTBOOLANDBOOLORNZNUMEQUALNUMNOTEQUALLTLEGTGEMINMAXWITHINPACKMAPPACKSTRUCTPACKUNPACKNEWARRAY0NEWARRAYNEWARRAY_TNEWSTRUCT0NEWSTRUCTNEWMAPSIZEHASKEYKEYSVALUESPICKITEMAPPENDSETITEMREVERSEITEMSREMOVECLEARITEMSPOPITEMISNULLISTYPECONVERT"
 
 var _Opcode_map = map[Opcode]string{
 	0:   _Opcode_name[0:8],
@@ -369,28 +371,30 @@ var _Opcode_map = map[Opcode]string{
 	185: _Opcode_name[923:926],
 	186: _Opcode_name[926:929],
 	187: _Opcode_name[929:935],
-	192: _Opcode_name[935:939],
+	190: _Opcode_name[935:942],
-	193: _Opcode_name[939:945],
+	191: _Opcode_name[942:952],
-	194: _Opcode_name[945:954],
+	192: _Opcode_name[952:956],
-	195: _Opcode_name[954:962],
+	193: _Opcode_name[956:962],
-	196: _Opcode_name[962:972],
+	194: _Opcode_name[962:971],
-	197: _Opcode_name[972:982],
+	195: _Opcode_name[971:979],
-	198: _Opcode_name[982:991],
+	196: _Opcode_name[979:989],
-	200: _Opcode_name[991:997],
+	197: _Opcode_name[989:999],
-	202: _Opcode_name[997:1001],
+	198: _Opcode_name[999:1008],
-	203: _Opcode_name[1001:1007],
+	200: _Opcode_name[1008:1014],
-	204: _Opcode_name[1007:1011],
+	202: _Opcode_name[1014:1018],
-	205: _Opcode_name[1011:1017],
+	203: _Opcode_name[1018:1024],
-	206: _Opcode_name[1017:1025],
+	204: _Opcode_name[1024:1028],
-	207: _Opcode_name[1025:1031],
+	205: _Opcode_name[1028:1034],
-	208: _Opcode_name[1031:1038],
+	206: _Opcode_name[1034:1042],
-	209: _Opcode_name[1038:1050],
+	207: _Opcode_name[1042:1048],
-	210: _Opcode_name[1050:1056],
+	208: _Opcode_name[1048:1055],
-	211: _Opcode_name[1056:1066],
+	209: _Opcode_name[1055:1067],
-	212: _Opcode_name[1066:1073],
+	210: _Opcode_name[1067:1073],
-	216: _Opcode_name[1073:1079],
+	211: _Opcode_name[1073:1083],
-	217: _Opcode_name[1079:1085],
+	212: _Opcode_name[1083:1090],
-	219: _Opcode_name[1085:1092],
+	216: _Opcode_name[1090:1096],
+	217: _Opcode_name[1096:1102],
+	219: _Opcode_name[1102:1109],
 }
 
 func (i Opcode) String() string {

pkg/vm/testdata/neo-vm

@@ -1 +1 @@
-Subproject commit 72e546b176401b85a03bc4653eeb177badb42d3b
+Subproject commit b18e040d2115ed2ea3c9a60ae8722a7865b38927

pkg/vm/vm.go

@@ -1061,7 +1061,23 @@ func (v *VM) execute(ctx *Context, op opcode.Opcode, parameter []byte) (err erro
 
 		v.refs.Add(val)
 
-	case opcode.PACK:
+	case opcode.PACKMAP:
+		n := toInt(v.estack.Pop().BigInt())
+		if n < 0 || n*2 > v.estack.Len() {
+			panic("invalid length")
+		}
+
+		items := make([]stackitem.MapElement, n)
+		for i := 0; i < n; i++ {
+			key := v.estack.Pop()
+			validateMapKey(key)
+			val := v.estack.Pop().value
+			items[i].Key = key.value
+			items[i].Value = val
+		}
+		v.estack.PushItem(stackitem.NewMapWithValue(items))
+
+	case opcode.PACKSTRUCT, opcode.PACK:
 		n := toInt(v.estack.Pop().BigInt())
 		if n < 0 || n > v.estack.Len() {
 			panic("OPACK: invalid length")
@@ -1072,13 +1088,39 @@ func (v *VM) execute(ctx *Context, op opcode.Opcode, parameter []byte) (err erro
 			items[i] = v.estack.Pop().value
 		}
 
-		v.estack.PushItem(stackitem.NewArray(items))
+		var res stackitem.Item
+		if op == opcode.PACK {
+			res = stackitem.NewArray(items)
+		} else {
+			res = stackitem.NewStruct(items)
+		}
+		v.estack.PushItem(res)
 
 	case opcode.UNPACK:
-		a := v.estack.Pop().Array()
+		e := v.estack.Pop()
-		l := len(a)
+		var arr []stackitem.Item
-		for i := l - 1; i >= 0; i-- {
+		var l int
-			v.estack.PushItem(a[i])
+
+		switch t := e.value.(type) {
+		case *stackitem.Array:
+			arr = t.Value().([]stackitem.Item)
+		case *stackitem.Struct:
+			arr = t.Value().([]stackitem.Item)
+		case *stackitem.Map:
+			m := t.Value().([]stackitem.MapElement)
+			l = len(m)
+			for i := l - 1; i >= 0; i-- {
+				v.estack.PushItem(m[i].Value)
+				v.estack.PushItem(m[i].Key)
+			}
+		default:
+			panic("element is not an array/struct/map")
+		}
+		if arr != nil {
+			l = len(arr)
+			for i := l - 1; i >= 0; i-- {
+				v.estack.PushItem(arr[i])
+			}
 		}
 		v.estack.PushItem(stackitem.NewBigInteger(big.NewInt(int64(l))))
 

pkg/vm/vm_test.go

@@ -1779,30 +1779,39 @@ func TestRIGHT(t *testing.T) {
 }
 
 func TestPACK(t *testing.T) {
-	prog := makeProgram(opcode.PACK)
+	for _, op := range []opcode.Opcode{opcode.PACK, opcode.PACKSTRUCT} {
-	t.Run("BadLen", getTestFuncForVM(prog, nil, 1))
+		t.Run(op.String(), func(t *testing.T) {
-	t.Run("Good0Len", getTestFuncForVM(prog, []stackitem.Item{}, 0))
+			prog := makeProgram(op)
+			t.Run("BadLen", getTestFuncForVM(prog, nil, 1))
+			t.Run("BigLen", getTestFuncForVM(prog, nil, 100500))
+			t.Run("Good0Len", getTestFuncForVM(prog, []stackitem.Item{}, 0))
+		})
+	}
 }
 
 func TestPACKGood(t *testing.T) {
-	prog := makeProgram(opcode.PACK)
+	for _, op := range []opcode.Opcode{opcode.PACK, opcode.PACKSTRUCT} {
-	elements := []int{55, 34, 42}
+		t.Run(op.String(), func(t *testing.T) {
-	vm := load(prog)
+			prog := makeProgram(op)
-	// canary
+			elements := []int{55, 34, 42}
-	vm.estack.PushVal(1)
+			vm := load(prog)
-	for i := len(elements) - 1; i >= 0; i-- {
+			// canary
-		vm.estack.PushVal(elements[i])
+			vm.estack.PushVal(1)
+			for i := len(elements) - 1; i >= 0; i-- {
+				vm.estack.PushVal(elements[i])
+			}
+			vm.estack.PushVal(len(elements))
+			runVM(t, vm)
+			assert.Equal(t, 2, vm.estack.Len())
+			a := vm.estack.Peek(0).Array()
+			assert.Equal(t, len(elements), len(a))
+			for i := 0; i < len(elements); i++ {
+				e := a[i].Value().(*big.Int)
+				assert.Equal(t, int64(elements[i]), e.Int64())
+			}
+			assert.Equal(t, int64(1), vm.estack.Peek(1).BigInt().Int64())
+		})
 	}
-	vm.estack.PushVal(len(elements))
-	runVM(t, vm)
-	assert.Equal(t, 2, vm.estack.Len())
-	a := vm.estack.Peek(0).Array()
-	assert.Equal(t, len(elements), len(a))
-	for i := 0; i < len(elements); i++ {
-		e := a[i].Value().(*big.Int)
-		assert.Equal(t, int64(elements[i]), e.Int64())
-	}
-	assert.Equal(t, int64(1), vm.estack.Peek(1).BigInt().Int64())
 }
 
 func TestPACK_UNPACK_MaxSize(t *testing.T) {
@@ -1851,6 +1860,42 @@ func TestPACK_UNPACK_PACK_MaxSize(t *testing.T) {
 	assert.Equal(t, int64(1), vm.estack.Peek(1).BigInt().Int64())
 }
 
+func TestPACKMAP_UNPACK_PACKMAP_MaxSize(t *testing.T) {
+	prog := makeProgram(opcode.PACKMAP, opcode.UNPACK, opcode.PACKMAP)
+	elements := make([]int, (MaxStackSize-2)/2)
+	vm := load(prog)
+	// canary
+	vm.estack.PushVal(-1)
+	for i := len(elements) - 1; i >= 0; i-- {
+		elements[i] = i
+		vm.estack.PushVal(i * 2)
+		vm.estack.PushVal(i)
+	}
+	vm.estack.PushVal(len(elements))
+	runVM(t, vm)
+	// check reference counter = 1+1+1024
+	assert.Equal(t, 1+1+len(elements), int(vm.refs))
+	assert.Equal(t, 2, vm.estack.Len())
+	m := vm.estack.Peek(0).value.(*stackitem.Map).Value().([]stackitem.MapElement)
+	assert.Equal(t, len(elements), len(m))
+	for i := 0; i < len(elements); i++ {
+		k := m[i].Key.Value().(*big.Int)
+		v := m[i].Value.Value().(*big.Int)
+		assert.Equal(t, int64(elements[i]), k.Int64())
+		assert.Equal(t, int64(elements[i])*2, v.Int64())
+	}
+	assert.Equal(t, int64(-1), vm.estack.Peek(1).BigInt().Int64())
+}
+
+func TestPACKMAPBadKey(t *testing.T) {
+	prog := makeProgram(opcode.PACKMAP)
+	vm := load(prog)
+	vm.estack.PushVal(1)
+	vm.estack.PushItem(stackitem.NewBuffer([]byte{1}))
+	vm.estack.PushVal(1)
+	checkVMFailed(t, vm)
+}
+
 func TestUNPACKBadNotArray(t *testing.T) {
 	prog := makeProgram(opcode.UNPACK)
 	runWithArgs(t, prog, nil, 1)