vm: fix a bug in collection serialize

When encountering already seen stack item we should fail
only if it is a collection. Duplicate Integers or ByteArrays are ok
because they can't lead to recursion.

pkg/vm/serialization.go

@@ -33,7 +33,6 @@ func serializeItemTo(item StackItem, w *io.BinWriter, seen map[StackItem]bool) {
 		w.Err = errors.New("recursive structures are not supported")
 		return
 	}
-	seen[item] = true
 
 	switch t := item.(type) {
 	case *ByteArrayItem:
@@ -48,6 +47,8 @@ func serializeItemTo(item StackItem, w *io.BinWriter, seen map[StackItem]bool) {
 	case *InteropItem:
 		w.Err = errors.New("not supported")
 	case *ArrayItem, *StructItem:
+		seen[item] = true
+
 		_, isArray := t.(*ArrayItem)
 		if isArray {
 			w.WriteLE(byte(arrayT))
@@ -61,6 +62,8 @@ func serializeItemTo(item StackItem, w *io.BinWriter, seen map[StackItem]bool) {
 			serializeItemTo(arr[i], w, seen)
 		}
 	case *MapItem:
+		seen[item] = true
+
 		w.WriteLE(byte(mapT))
 		w.WriteVarUint(uint64(len(t.value)))
 		for k, v := range t.value {

pkg/vm/vm_test.go

@@ -379,6 +379,17 @@ func TestSerializeArrayBad(t *testing.T) {
 	require.True(t, vm.HasFailed())
 }
 
+func TestSerializeDupInteger(t *testing.T) {
+	prog := []byte{
+		byte(PUSH0), byte(NEWARRAY),
+		byte(DUP), byte(PUSH2), byte(DUP), byte(TOALTSTACK), byte(APPEND),
+		byte(DUP), byte(FROMALTSTACK), byte(APPEND),
+	}
+	vm := load(append(prog, getSerializeProg()...))
+
+	runVM(t, vm)
+}
+
 func TestSerializeStruct(t *testing.T) {
 	vm := load(getSerializeProg())
 	item := NewStructItem([]StackItem{