stackitem: cache visited items while marshaling json

```
name      old time/op    new time/op    delta
ToJSON-8    4.52ms ± 4%    0.05ms ±34%  -98.89%  (p=0.000 n=8+10)

name      old alloc/op   new alloc/op   delta
ToJSON-8    2.13MB ± 0%    0.40MB ± 0%  -81.44%  (p=0.000 n=9+6)

name      old allocs/op  new allocs/op  delta
ToJSON-8     65.6k ± 0%      0.0k ± 0%  -99.95%  (p=0.000 n=10+10)
```

Signed-off-by: Evgeniy Stratonikov <evgeniy@nspcc.ru>
This commit is contained in:
Evgeniy Stratonikov 2021-07-09 14:04:36 +03:00
parent 69cdd5252a
commit dc0a17dd0e

View file

@ -9,8 +9,6 @@ import (
gio "io" gio "io"
"math" "math"
"math/big" "math/big"
"github.com/nspcc-dev/neo-go/pkg/io"
) )
// decoder is a wrapper around json.Decoder helping to mimic C# json decoder behaviour. // decoder is a wrapper around json.Decoder helping to mimic C# json decoder behaviour.
@ -43,87 +41,106 @@ var ErrTooDeep = errors.New("too deep")
// Array, Struct -> array // Array, Struct -> array
// Map -> map with keys as UTF-8 bytes // Map -> map with keys as UTF-8 bytes
func ToJSON(item Item) ([]byte, error) { func ToJSON(item Item) ([]byte, error) {
buf := io.NewBufBinWriter() seen := make(map[Item]sliceNoPointer)
toJSON(buf, item) return toJSON(nil, seen, item)
if buf.Err != nil {
return nil, buf.Err
}
return buf.Bytes(), nil
} }
func toJSON(buf *io.BufBinWriter, item Item) { // sliceNoPointer represents sub-slice of a known slice.
w := buf.BinWriter // It doesn't contain pointer and uses less memory than `[]byte`.
if w.Err != nil { type sliceNoPointer struct {
return start, end int
} else if buf.Len() > MaxSize {
w.Err = errTooBigSize
} }
func toJSON(data []byte, seen map[Item]sliceNoPointer, item Item) ([]byte, error) {
if len(data) > MaxSize {
return nil, errTooBigSize
}
if old, ok := seen[item]; ok {
if len(data)+old.end-old.start > MaxSize {
return nil, errTooBigSize
}
return append(data, data[old.start:old.end]...), nil
}
start := len(data)
var err error
switch it := item.(type) { switch it := item.(type) {
case *Array, *Struct: case *Array, *Struct:
w.WriteB('[') data = append(data, '[')
items := it.Value().([]Item) items := it.Value().([]Item)
for i, v := range items { for i, v := range items {
toJSON(buf, v) data, err = toJSON(data, seen, v)
if err != nil {
return nil, err
}
if i < len(items)-1 { if i < len(items)-1 {
w.WriteB(',') data = append(data, ',')
} }
} }
w.WriteB(']') data = append(data, ']')
seen[item] = sliceNoPointer{start, len(data)}
case *Map: case *Map:
w.WriteB('{') data = append(data, '{')
for i := range it.value { for i := range it.value {
// map key can always be converted to []byte // map key can always be converted to []byte
// but are not always a valid UTF-8. // but are not always a valid UTF-8.
writeJSONString(buf.BinWriter, it.value[i].Key) raw, err := itemToJSONString(it.value[i].Key)
w.WriteBytes([]byte(`:`)) if err != nil {
toJSON(buf, it.value[i].Value) return nil, err
}
data = append(data, raw...)
data = append(data, ':')
data, err = toJSON(data, seen, it.value[i].Value)
if err != nil {
return nil, err
}
if i < len(it.value)-1 { if i < len(it.value)-1 {
w.WriteB(',') data = append(data, ',')
} }
} }
w.WriteB('}') data = append(data, '}')
seen[item] = sliceNoPointer{start, len(data)}
case *BigInteger: case *BigInteger:
if it.value.CmpAbs(big.NewInt(MaxAllowedInteger)) == 1 { if it.value.CmpAbs(big.NewInt(MaxAllowedInteger)) == 1 {
w.Err = fmt.Errorf("%w (MaxAllowedInteger)", ErrInvalidValue) return nil, fmt.Errorf("%w (MaxAllowedInteger)", ErrInvalidValue)
return
} }
w.WriteBytes([]byte(it.value.String())) data = append(data, it.value.String()...)
case *ByteArray, *Buffer: case *ByteArray, *Buffer:
writeJSONString(w, it) raw, err := itemToJSONString(it)
if err != nil {
return nil, err
}
data = append(data, raw...)
case *Bool: case *Bool:
if it.value { if it.value {
w.WriteBytes([]byte("true")) data = append(data, "true"...)
} else { } else {
w.WriteBytes([]byte("false")) data = append(data, "false"...)
} }
case Null: case Null:
w.WriteBytes([]byte("null")) data = append(data, "null"...)
default: default:
w.Err = fmt.Errorf("%w: %s", ErrUnserializable, it.String()) return nil, fmt.Errorf("%w: %s", ErrUnserializable, it.String())
return
} }
if w.Err == nil && buf.Len() > MaxSize { if len(data) > MaxSize {
w.Err = errTooBigSize return nil, errTooBigSize
} }
return data, nil
} }
// writeJSONString converts it to string and writes it to w as JSON value // itemToJSONString converts it to string
// surrounded in quotes with control characters escaped. // surrounded in quotes with control characters escaped.
func writeJSONString(w *io.BinWriter, it Item) { func itemToJSONString(it Item) ([]byte, error) {
if w.Err != nil {
return
}
s, err := ToString(it) s, err := ToString(it)
if err != nil { if err != nil {
w.Err = err return nil, err
return
} }
data, _ := json.Marshal(s) // error never occurs because `ToString` checks for validity data, _ := json.Marshal(s) // error never occurs because `ToString` checks for validity
// ref https://github.com/neo-project/neo-modules/issues/375 and https://github.com/dotnet/runtime/issues/35281 // ref https://github.com/neo-project/neo-modules/issues/375 and https://github.com/dotnet/runtime/issues/35281
data = bytes.Replace(data, []byte{'+'}, []byte("\\u002B"), -1) return bytes.Replace(data, []byte{'+'}, []byte("\\u002B"), -1), nil
w.WriteBytes(data)
} }
// FromJSON decodes Item from JSON. // FromJSON decodes Item from JSON.