From d836233352e71437acfe7fed2e9f0e357d0f8f9b Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Thu, 11 Jun 2020 16:27:57 +0300 Subject: [PATCH] stackitem: allow to (de-)serialize items to JSON This commit implements behavior identical to that of C# `System.Json.*` interops. --- pkg/vm/stackitem/json.go | 202 ++++++++++++++++++++++++++++++++++ pkg/vm/stackitem/json_test.go | 105 ++++++++++++++++++ 2 files changed, 307 insertions(+) create mode 100644 pkg/vm/stackitem/json.go create mode 100644 pkg/vm/stackitem/json_test.go diff --git a/pkg/vm/stackitem/json.go b/pkg/vm/stackitem/json.go new file mode 100644 index 000000000..81b67f769 --- /dev/null +++ b/pkg/vm/stackitem/json.go @@ -0,0 +1,202 @@ +package stackitem + +import ( + "bytes" + "encoding/base64" + "encoding/json" + "errors" + "fmt" + gio "io" + "math" + "math/big" + + "github.com/nspcc-dev/neo-go/pkg/io" +) + +// decoder is a wrapper around json.Decoder helping to mimic C# json decoder behaviour. +type decoder struct { + json.Decoder + + depth int +} + +// MaxAllowedInteger is the maximum integer allowed to be encoded. +const MaxAllowedInteger = 2<<53 - 1 + +// maxJSONDepth is a maximum allowed depth-level of decoded JSON. +const maxJSONDepth = 10 + +// ToJSON encodes Item to JSON. +// It behaves as following: +// ByteArray -> base64 string +// BigInteger -> number +// Bool -> bool +// Null -> null +// Array, Struct -> array +// Map -> map with keys as UTF-8 bytes +func ToJSON(item Item) ([]byte, error) { + buf := io.NewBufBinWriter() + toJSON(buf, item) + if buf.Err != nil { + return nil, buf.Err + } + return buf.Bytes(), nil +} + +func toJSON(buf *io.BufBinWriter, item Item) { + w := buf.BinWriter + if w.Err != nil { + return + } else if buf.Len() > MaxSize { + w.Err = errors.New("item is too big") + } + switch it := item.(type) { + case *Array, *Struct: + w.WriteB('[') + items := it.Value().([]Item) + for i, v := range items { + toJSON(buf, v) + if i < len(items)-1 { + w.WriteB(',') + } + } + w.WriteB(']') + case *Map: + w.WriteB('{') + for i := range it.value { + bs, _ := it.value[i].Key.TryBytes() // map key can always be converted to []byte + w.WriteB('"') + w.WriteBytes(bs) + w.WriteBytes([]byte(`":`)) + toJSON(buf, it.value[i].Value) + if i < len(it.value)-1 { + w.WriteB(',') + } + } + w.WriteB('}') + case *BigInteger: + if it.value.CmpAbs(big.NewInt(MaxAllowedInteger)) == 1 { + w.Err = errors.New("too big integer") + return + } + w.WriteBytes([]byte(it.value.String())) + case *ByteArray: + w.WriteB('"') + val := it.Value().([]byte) + b := make([]byte, base64.StdEncoding.EncodedLen(len(val))) + base64.StdEncoding.Encode(b, val) + w.WriteBytes(b) + w.WriteB('"') + case *Bool: + if it.value { + w.WriteBytes([]byte("true")) + } else { + w.WriteBytes([]byte("false")) + } + case Null: + w.WriteBytes([]byte("null")) + default: + w.Err = fmt.Errorf("invalid item: %s", it.String()) + return + } + if w.Err == nil && buf.Len() > MaxSize { + w.Err = errors.New("item is too big") + } +} + +// FromJSON decodes Item from JSON. +// It behaves as following: +// string -> ByteArray from base64 +// number -> BigInteger +// bool -> Bool +// null -> Null +// array -> Array +// map -> Map, keys are UTF-8 +func FromJSON(data []byte) (Item, error) { + d := decoder{Decoder: *json.NewDecoder(bytes.NewReader(data))} + if item, err := d.decode(); err != nil { + return nil, err + } else if _, err := d.Token(); err != gio.EOF { + return nil, errors.New("unexpected items") + } else { + return item, nil + } +} + +func (d *decoder) decode() (Item, error) { + tok, err := d.Token() + if err != nil { + return nil, err + } + switch t := tok.(type) { + case json.Delim: + switch t { + case json.Delim('{'), json.Delim('['): + if d.depth == maxJSONDepth { + return nil, errors.New("JSON depth limit exceeded") + } + d.depth++ + var item Item + if t == json.Delim('{') { + item, err = d.decodeMap() + } else { + item, err = d.decodeArray() + } + d.depth-- + return item, err + default: + // no error above means corresponding closing token + // was encountered for map or array respectively + return nil, nil + } + case string: + b, err := base64.StdEncoding.DecodeString(t) + if err != nil { + return nil, err + } + return NewByteArray(b), nil + case float64: + if math.Floor(t) != t { + return nil, fmt.Errorf("real value is not allowed: %v", t) + } + return NewBigInteger(big.NewInt(int64(t))), nil + case bool: + return NewBool(t), nil + default: + // it can be only `nil` + return Null{}, nil + } +} + +func (d *decoder) decodeArray() (*Array, error) { + items := []Item{} + for { + item, err := d.decode() + if err != nil { + return nil, err + } + if item == nil { + return NewArray(items), nil + } + items = append(items, item) + } +} + +func (d *decoder) decodeMap() (*Map, error) { + m := NewMap() + for { + key, err := d.Token() + if err != nil { + return nil, err + } + k, ok := key.(string) + if !ok { + return m, nil + } + val, err := d.decode() + if err != nil { + return nil, err + } + m.Add(NewByteArray([]byte(k)), val) + } +} diff --git a/pkg/vm/stackitem/json_test.go b/pkg/vm/stackitem/json_test.go new file mode 100644 index 000000000..1b1ef7718 --- /dev/null +++ b/pkg/vm/stackitem/json_test.go @@ -0,0 +1,105 @@ +package stackitem + +import ( + "encoding/base64" + "math/big" + "testing" + + "github.com/stretchr/testify/require" +) + +func getTestDecodeFunc(js string, expected ...interface{}) func(t *testing.T) { + return func(t *testing.T) { + actual, err := FromJSON([]byte(js)) + if expected[0] == nil { + require.Error(t, err) + return + } + require.NoError(t, err) + require.Equal(t, Make(expected[0]), actual) + + if len(expected) == 1 { + encoded, err := ToJSON(actual) + require.NoError(t, err) + require.Equal(t, js, string(encoded)) + } + } +} + +func TestFromToJSON(t *testing.T) { + var testBase64 = base64.StdEncoding.EncodeToString([]byte("test")) + t.Run("ByteString", func(t *testing.T) { + t.Run("Empty", getTestDecodeFunc(`""`, []byte{})) + t.Run("Base64", getTestDecodeFunc(`"`+testBase64+`"`, "test")) + }) + t.Run("BigInteger", func(t *testing.T) { + t.Run("ZeroFloat", getTestDecodeFunc(`12.000`, 12, nil)) + t.Run("NonZeroFloat", getTestDecodeFunc(`12.01`, nil)) + t.Run("Negative", getTestDecodeFunc(`-4`, -4)) + t.Run("Positive", getTestDecodeFunc(`123`, 123)) + }) + t.Run("Bool", func(t *testing.T) { + t.Run("True", getTestDecodeFunc(`true`, true)) + t.Run("False", getTestDecodeFunc(`false`, false)) + }) + t.Run("Null", getTestDecodeFunc(`null`, Null{})) + t.Run("Array", func(t *testing.T) { + t.Run("Empty", getTestDecodeFunc(`[]`, NewArray([]Item{}))) + t.Run("Simple", getTestDecodeFunc((`[1,"`+testBase64+`",true,null]`), + NewArray([]Item{NewBigInteger(big.NewInt(1)), NewByteArray([]byte("test")), NewBool(true), Null{}}))) + t.Run("Nested", getTestDecodeFunc(`[[],[{},null]]`, + NewArray([]Item{NewArray([]Item{}), NewArray([]Item{NewMap(), Null{}})}))) + }) + t.Run("Map", func(t *testing.T) { + small := NewMap() + small.Add(NewByteArray([]byte("a")), NewBigInteger(big.NewInt(3))) + large := NewMap() + large.Add(NewByteArray([]byte("3")), small) + large.Add(NewByteArray([]byte("arr")), NewArray([]Item{NewByteArray([]byte("test"))})) + t.Run("Empty", getTestDecodeFunc(`{}`, NewMap())) + t.Run("Small", getTestDecodeFunc(`{"a":3}`, small)) + t.Run("Big", getTestDecodeFunc(`{"3":{"a":3},"arr":["`+testBase64+`"]}`, large)) + }) + t.Run("Invalid", func(t *testing.T) { + t.Run("Empty", getTestDecodeFunc(``, nil)) + t.Run("InvalidString", getTestDecodeFunc(`"not a base64"`, nil)) + t.Run("InvalidArray", getTestDecodeFunc(`[}`, nil)) + t.Run("InvalidMap", getTestDecodeFunc(`{]`, nil)) + t.Run("InvalidMapValue", getTestDecodeFunc(`{"a":{]}`, nil)) + t.Run("AfterArray", getTestDecodeFunc(`[]XX`, nil)) + t.Run("EncodeBigInteger", func(t *testing.T) { + item := NewBigInteger(big.NewInt(MaxAllowedInteger + 1)) + _, err := ToJSON(item) + require.Error(t, err) + }) + t.Run("EncodeInvalidItemType", func(t *testing.T) { + item := NewPointer(1, []byte{1, 2, 3}) + _, err := ToJSON(item) + require.Error(t, err) + }) + t.Run("BigByteArray", func(t *testing.T) { + l := base64.StdEncoding.DecodedLen(MaxSize + 8) + require.True(t, l < MaxSize) // check if test makes sense + item := NewByteArray(make([]byte, l)) + _, err := ToJSON(item) + require.Error(t, err) + }) + t.Run("BigNestedArray", getTestDecodeFunc(`[[[[[[[[[[[]]]]]]]]]]]`, nil)) + t.Run("EncodeRecursive", func(t *testing.T) { + // add this item to speed up test a bit + item := NewByteArray(make([]byte, MaxSize/100)) + t.Run("Array", func(t *testing.T) { + arr := NewArray([]Item{item}) + arr.Append(arr) + _, err := ToJSON(arr) + require.Error(t, err) + }) + t.Run("Map", func(t *testing.T) { + m := NewMap() + m.Add(item, m) + _, err := ToJSON(m) + require.Error(t, err) + }) + }) + }) +}