stackitem: add limit to serialized items

Standard binary serialization/deserialization is mostly used in VM to put/get
elements into/from storage, so they never should exceed MaxSize (otherwise one
won't be able to deserialize these items).

This patch leaves EncodeBinaryStackItem unprotected, but that's a streaming
interface, so it's up to the user of it to ensure its appropriate use (and our
uses are mostly for native contract's data, so they're fine).

pkg/vm/stackitem/serialization.go

@@ -12,6 +12,7 @@ import (
 // serContext is an internal serialization context.
 type serContext struct {
 	*io.BinWriter
+	buf          *io.BufBinWriter
 	allowInvalid bool
 	seen         map[Item]bool
 }
@@ -21,6 +22,7 @@ func SerializeItem(item Item) ([]byte, error) {
 	w := io.NewBufBinWriter()
 	sc := serContext{
 		BinWriter:    w.BinWriter,
+		buf:          w,
 		allowInvalid: false,
 		seen:         make(map[Item]bool),
 	}
@@ -49,6 +51,7 @@ func EncodeBinaryStackItemAppExec(item Item, w *io.BinWriter) {
 	bw := io.NewBufBinWriter()
 	sc := serContext{
 		BinWriter:    bw.BinWriter,
+		buf:          bw,
 		allowInvalid: true,
 		seen:         make(map[Item]bool),
 	}
@@ -68,10 +71,6 @@ func (w *serContext) serialize(item Item) {
 		w.Err = errors.New("recursive structures can't be serialized")
 		return
 	}
-	if item == nil && w.allowInvalid {
-		w.WriteBytes([]byte{byte(InvalidT)})
-		return
-	}
 
 	switch t := item.(type) {
 	case *ByteArray:
@@ -120,6 +119,16 @@ func (w *serContext) serialize(item Item) {
 		delete(w.seen, item)
 	case Null:
 		w.WriteB(byte(AnyT))
+	case nil:
+		if w.allowInvalid {
+			w.WriteBytes([]byte{byte(InvalidT)})
+		} else {
+			w.Err = errors.New("invalid stack item")
+		}
+	}
+
+	if w.Err == nil && w.buf != nil && w.buf.Len() > MaxSize {
+		w.Err = errors.New("too big item")
 	}
 }
 

pkg/vm/stackitem/serialization_test.go

@@ -0,0 +1,21 @@
+package stackitem
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestSerializationMaxErr(t *testing.T) {
+	base := make([]byte, MaxSize/2+1)
+	item := Make(base)
+
+	arr := []Item{item, item.Dup()}
+	aitem := Make(arr)
+
+	_, err := SerializeItem(item)
+	require.NoError(t, err)
+
+	_, err = SerializeItem(aitem)
+	require.Error(t, err)
+}