Merge pull request #1871 from nspcc-dev/mpt/serialisation-fix

mpt: refactor serialisation
This commit is contained in:
Roman Khimov 2021-04-01 11:01:38 +03:00 committed by GitHub
commit 2c8f8bf3ec
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
14 changed files with 110 additions and 105 deletions

View file

@ -9,6 +9,7 @@ import (
"github.com/nspcc-dev/neo-go/pkg/core/block"
"github.com/nspcc-dev/neo-go/pkg/core/interop"
"github.com/nspcc-dev/neo-go/pkg/core/native"
"github.com/nspcc-dev/neo-go/pkg/core/storage"
"github.com/nspcc-dev/neo-go/pkg/core/transaction"
"github.com/nspcc-dev/neo-go/pkg/crypto/hash"
"github.com/nspcc-dev/neo-go/pkg/crypto/keys"
@ -16,14 +17,6 @@ import (
"github.com/nspcc-dev/neo-go/pkg/vm/stackitem"
)
const (
// MaxStorageKeyLen is the maximum length of a key for storage items.
MaxStorageKeyLen = 64
// MaxStorageValueLen is the maximum length of a value for storage items.
// It is set to be the maximum value for uint16.
MaxStorageValueLen = 65535
)
// StorageContext contains storing id and read/write flag, it's used as
// a context for storage manipulation functions.
type StorageContext struct {
@ -31,17 +24,6 @@ type StorageContext struct {
ReadOnly bool
}
// StorageFlag represents storage flag which denotes whether the stored value is
// a constant.
type StorageFlag byte
const (
// None is a storage flag for non-constant items.
None StorageFlag = 0
// Constant is a storage flag for constant items.
Constant StorageFlag = 0x01
)
// engineGetScriptContainer returns transaction or block that contains the script
// being run.
func engineGetScriptContainer(ic *interop.Context) error {
@ -115,10 +97,10 @@ func storageGetContextInternal(ic *interop.Context, isReadOnly bool) error {
}
func putWithContext(ic *interop.Context, stc *StorageContext, key []byte, value []byte) error {
if len(key) > MaxStorageKeyLen {
if len(key) > storage.MaxStorageKeyLen {
return errors.New("key is too big")
}
if len(value) > MaxStorageValueLen {
if len(value) > storage.MaxStorageValueLen {
return errors.New("value is too big")
}
if stc.ReadOnly {

View file

@ -13,6 +13,7 @@ import (
"github.com/nspcc-dev/neo-go/pkg/core/interop/runtime"
"github.com/nspcc-dev/neo-go/pkg/core/native"
"github.com/nspcc-dev/neo-go/pkg/core/state"
"github.com/nspcc-dev/neo-go/pkg/core/storage"
"github.com/nspcc-dev/neo-go/pkg/core/transaction"
"github.com/nspcc-dev/neo-go/pkg/crypto/hash"
"github.com/nspcc-dev/neo-go/pkg/crypto/keys"
@ -198,7 +199,7 @@ func TestStoragePut(t *testing.T) {
})
t.Run("check limits", func(t *testing.T) {
initVM(t, make([]byte, MaxStorageKeyLen), make([]byte, MaxStorageValueLen), -1)
initVM(t, make([]byte, storage.MaxStorageKeyLen), make([]byte, storage.MaxStorageValueLen), -1)
require.NoError(t, storagePut(ic))
})
@ -209,11 +210,11 @@ func TestStoragePut(t *testing.T) {
require.Error(t, storagePut(ic))
})
t.Run("big key", func(t *testing.T) {
initVM(t, make([]byte, MaxStorageKeyLen+1), []byte{1}, -1)
initVM(t, make([]byte, storage.MaxStorageKeyLen+1), []byte{1}, -1)
require.Error(t, storagePut(ic))
})
t.Run("big value", func(t *testing.T) {
initVM(t, []byte{1}, make([]byte, MaxStorageValueLen+1), -1)
initVM(t, []byte{1}, make([]byte, storage.MaxStorageValueLen+1), -1)
require.Error(t, storagePut(ic))
})
})

View file

@ -23,6 +23,7 @@ type BaseNodeIface interface {
Hash() util.Uint256
Type() NodeType
Bytes() []byte
EncodeBinaryAsChild(w *io.BinWriter)
}
type flushedNode interface {
@ -77,7 +78,17 @@ func (b *BaseNode) invalidateCache() {
// encodeNodeWithType encodes node together with it's type.
func encodeNodeWithType(n Node, w *io.BinWriter) {
w.WriteB(byte(n.Type()))
switch t := n.Type(); t {
case HashT:
hn := n.(*HashNode)
if !hn.hashValid {
w.WriteB(byte(EmptyT))
break
}
fallthrough
default:
w.WriteB(byte(t))
}
n.EncodeBinary(w)
}
@ -93,9 +104,19 @@ func DecodeNodeWithType(r *io.BinReader) Node {
case ExtensionT:
n = new(ExtensionNode)
case HashT:
n = new(HashNode)
n = &HashNode{
BaseNode: BaseNode{
hashValid: true,
},
}
case LeafT:
n = new(LeafNode)
case EmptyT:
n = &HashNode{
BaseNode: BaseNode{
hashValid: false,
},
}
default:
r.Err = fmt.Errorf("invalid node type: %x", typ)
return nil

View file

@ -48,20 +48,22 @@ func (b *BranchNode) Bytes() []byte {
// EncodeBinary implements io.Serializable.
func (b *BranchNode) EncodeBinary(w *io.BinWriter) {
for i := 0; i < childrenCount; i++ {
if hn, ok := b.Children[i].(*HashNode); ok {
hn.EncodeBinary(w)
continue
}
n := NewHashNode(b.Children[i].Hash())
n.EncodeBinary(w)
b.Children[i].EncodeBinaryAsChild(w)
}
}
// EncodeBinaryAsChild implements BaseNode interface.
func (b *BranchNode) EncodeBinaryAsChild(w *io.BinWriter) {
n := &NodeObject{Node: NewHashNode(b.Hash())} // with type
n.EncodeBinary(w)
}
// DecodeBinary implements io.Serializable.
func (b *BranchNode) DecodeBinary(r *io.BinReader) {
for i := 0; i < childrenCount; i++ {
b.Children[i] = new(HashNode)
b.Children[i].DecodeBinary(r)
no := new(NodeObject)
no.DecodeBinary(r)
b.Children[i] = no.Node
}
}

View file

@ -6,12 +6,13 @@ import (
"errors"
"fmt"
"github.com/nspcc-dev/neo-go/pkg/core/storage"
"github.com/nspcc-dev/neo-go/pkg/io"
"github.com/nspcc-dev/neo-go/pkg/util"
)
// MaxKeyLength is the max length of the extension node key.
const MaxKeyLength = 1125
const MaxKeyLength = (storage.MaxStorageKeyLen + 4) * 2
// ExtensionNode represents MPT's extension node.
type ExtensionNode struct {
@ -53,15 +54,21 @@ func (e *ExtensionNode) DecodeBinary(r *io.BinReader) {
}
e.key = make([]byte, sz)
r.ReadBytes(e.key)
e.next = new(HashNode)
e.next.DecodeBinary(r)
no := new(NodeObject)
no.DecodeBinary(r)
e.next = no.Node
e.invalidateCache()
}
// EncodeBinary implements io.Serializable.
func (e ExtensionNode) EncodeBinary(w *io.BinWriter) {
w.WriteVarBytes(e.key)
n := NewHashNode(e.next.Hash())
e.next.EncodeBinaryAsChild(w)
}
// EncodeBinaryAsChild implements BaseNode interface.
func (e *ExtensionNode) EncodeBinaryAsChild(w *io.BinWriter) {
n := &NodeObject{Node: NewHashNode(e.Hash())} // with type
n.EncodeBinary(w)
}

View file

@ -2,7 +2,6 @@ package mpt
import (
"errors"
"fmt"
"github.com/nspcc-dev/neo-go/pkg/io"
"github.com/nspcc-dev/neo-go/pkg/util"
@ -46,25 +45,23 @@ func (h *HashNode) Bytes() []byte {
// DecodeBinary implements io.Serializable.
func (h *HashNode) DecodeBinary(r *io.BinReader) {
sz := r.ReadVarUint()
switch sz {
case 0:
h.hashValid = false
case util.Uint256Size:
h.hashValid = true
r.ReadBytes(h.hash[:])
default:
r.Err = fmt.Errorf("invalid hash node size: %d", sz)
if h.hashValid {
h.hash.DecodeBinary(r)
}
}
// EncodeBinary implements io.Serializable.
func (h HashNode) EncodeBinary(w *io.BinWriter) {
if !h.hashValid {
w.WriteVarUint(0)
return
}
w.WriteVarBytes(h.hash[:])
w.WriteBytes(h.hash[:])
}
// EncodeBinaryAsChild implements BaseNode interface.
func (h *HashNode) EncodeBinaryAsChild(w *io.BinWriter) {
no := &NodeObject{Node: h} // with type
no.EncodeBinary(w)
}
// MarshalJSON implements json.Marshaler.

View file

@ -5,12 +5,13 @@ import (
"errors"
"fmt"
"github.com/nspcc-dev/neo-go/pkg/core/storage"
"github.com/nspcc-dev/neo-go/pkg/io"
"github.com/nspcc-dev/neo-go/pkg/util"
)
// MaxValueLength is a max length of a leaf node value.
const MaxValueLength = 1024 * 1024
const MaxValueLength = 3 + storage.MaxStorageValueLen + 1
// LeafNode represents MPT's leaf node.
type LeafNode struct {
@ -55,6 +56,12 @@ func (n LeafNode) EncodeBinary(w *io.BinWriter) {
w.WriteVarBytes(n.value)
}
// EncodeBinaryAsChild implements BaseNode interface.
func (n *LeafNode) EncodeBinaryAsChild(w *io.BinWriter) {
no := &NodeObject{Node: NewHashNode(n.Hash())} // with type
no.EncodeBinary(w)
}
// MarshalJSON implements json.Marshaler.
func (n *LeafNode) MarshalJSON() ([]byte, error) {
return []byte(`{"value":"` + hex.EncodeToString(n.value) + `"}`), nil

View file

@ -16,8 +16,9 @@ type NodeType byte
const (
BranchT NodeType = 0x00
ExtensionT NodeType = 0x01
HashT NodeType = 0x02
LeafT NodeType = 0x03
LeafT NodeType = 0x02
HashT NodeType = 0x03
EmptyT NodeType = 0x04
)
// NodeObject represents Node together with it's type.

View file

@ -16,6 +16,9 @@ func getTestFuncEncode(ok bool, expected, actual Node) func(t *testing.T) {
t.Run("IO", func(t *testing.T) {
bs, err := testserdes.EncodeBinary(expected)
require.NoError(t, err)
if hn, ok := actual.(*HashNode); ok {
hn.hashValid = true // this field is set during NodeObject decoding
}
err = testserdes.DecodeBinary(bs, actual)
if !ok {
require.Error(t, err)
@ -80,8 +83,8 @@ func TestNode_Serializable(t *testing.T) {
})
t.Run("InvalidSize", func(t *testing.T) {
buf := io.NewBufBinWriter()
buf.BinWriter.WriteVarBytes(make([]byte, 13))
require.Error(t, testserdes.DecodeBinary(buf.Bytes(), new(HashNode)))
buf.BinWriter.WriteBytes(make([]byte, 13))
require.Error(t, testserdes.DecodeBinary(buf.Bytes(), &HashNode{BaseNode: BaseNode{hashValid: true}}))
})
})
@ -151,6 +154,6 @@ func TestRootHash(t *testing.T) {
b.Children[9] = l2
r1 := NewExtensionNode([]byte{0x0A, 0x0C, 0x00, 0x01}, v1)
require.Equal(t, "dea3ab46e9461e885ed7091c1e533e0a8030b248d39cbc638962394eaca0fbb3", r1.Hash().StringLE())
require.Equal(t, "93e8e1ffe2f83dd92fca67330e273bcc811bf64b8f8d9d1b25d5e7366b47d60d", r.Hash().StringLE())
require.Equal(t, "cedd9897dd1559fbd5dfe5cfb223464da6de438271028afb8d647e950cbd18e0", r1.Hash().StringLE())
require.Equal(t, "1037e779c8a0313bd0d99c4151fa70a277c43c53a549b6444079f2e67e8ffb7b", r.Hash().StringLE())
}

View file

@ -23,6 +23,14 @@ const (
SYSVersion KeyPrefix = 0xf0
)
const (
// MaxStorageKeyLen is the maximum length of a key for storage items.
MaxStorageKeyLen = 64
// MaxStorageValueLen is the maximum length of a value for storage items.
// It is set to be the maximum value for uint16.
MaxStorageValueLen = 65535
)
// ErrKeyNotFound is an error returned by Store implementations
// when a certain key is not found.
var ErrKeyNotFound = errors.New("key not found")

View file

@ -2,9 +2,8 @@ package result
import (
"bytes"
"encoding/hex"
"encoding/base64"
"encoding/json"
"errors"
"github.com/nspcc-dev/neo-go/pkg/io"
)
@ -21,12 +20,6 @@ type ProofWithKey struct {
Proof [][]byte
}
// GetProof is a result of getproof RPC.
type GetProof struct {
Result ProofWithKey `json:"proof"`
Success bool `json:"success"`
}
// VerifyProof is a result of verifyproof RPC.
// nil Value is considered invalid.
type VerifyProof struct {
@ -40,7 +33,7 @@ func (p *ProofWithKey) MarshalJSON() ([]byte, error) {
if w.Err != nil {
return nil, w.Err
}
return []byte(`"` + hex.EncodeToString(w.Bytes()) + `"`), nil
return []byte(`"` + base64.StdEncoding.EncodeToString(w.Bytes()) + `"`), nil
}
// EncodeBinary implements io.Serializable.
@ -74,12 +67,12 @@ func (p *ProofWithKey) UnmarshalJSON(data []byte) error {
func (p *ProofWithKey) String() string {
w := io.NewBufBinWriter()
p.EncodeBinary(w.BinWriter)
return hex.EncodeToString(w.Bytes())
return base64.StdEncoding.EncodeToString(w.Bytes())
}
// FromString decodes p from hex-encoded string.
func (p *ProofWithKey) FromString(s string) error {
rawProof, err := hex.DecodeString(s)
rawProof, err := base64.StdEncoding.DecodeString(s)
if err != nil {
return err
}
@ -93,7 +86,7 @@ func (p *VerifyProof) MarshalJSON() ([]byte, error) {
if p.Value == nil {
return []byte(`"invalid"`), nil
}
return []byte(`{"value":"` + hex.EncodeToString(p.Value) + `"}`), nil
return []byte(`"` + base64.StdEncoding.EncodeToString(p.Value) + `"`), nil
}
// UnmarshalJSON implements json.Unmarshaler.
@ -102,18 +95,11 @@ func (p *VerifyProof) UnmarshalJSON(data []byte) error {
p.Value = nil
return nil
}
var m map[string]string
var m string
if err := json.Unmarshal(data, &m); err != nil {
return err
}
if len(m) != 1 {
return errors.New("must have single key")
}
v, ok := m["value"]
if !ok {
return errors.New("invalid json")
}
b, err := hex.DecodeString(v)
b, err := base64.StdEncoding.DecodeString(m)
if err != nil {
return err
}

View file

@ -24,23 +24,17 @@ func testProofWithKey() *ProofWithKey {
func TestGetProof_MarshalJSON(t *testing.T) {
t.Run("Good", func(t *testing.T) {
p := &GetProof{
Result: *testProofWithKey(),
Success: true,
}
testserdes.MarshalUnmarshalJSON(t, p, new(GetProof))
p := testProofWithKey()
testserdes.MarshalUnmarshalJSON(t, p, new(ProofWithKey))
})
t.Run("Compatibility", func(t *testing.T) {
js := []byte(`{
"proof" : "25ddeb9aa1bfc353c9c54e21dffb470f65d9c22a0662616c616e63654f70000000000000000708fd12020020666eaa8a6e75d43a97d76e72b605c7e05189f0c57ec19d84acdb75810f18239d202c83028ce3d7abcf4e4f95d05fbfdfa5e18bde3a8fbb65a57559d6b5ea09425c2090c40d440744a848e3b407a00e4efb692a957245a1efc9cb8496cb05fd328ee620dd2652bf25dfc3ad5fee7b200ccf3e3ae50772ff8ed58907e4dab8e7d4b2489720d8a5d5ed75b5b0f256d0a2cf5c220b4ddae2a228ef0fc0212b689f3811dfa94620342cc0d73fabd2440ed2cc735a9608391a510e1981b321a9f4258682706adc9620ced036e52f39387b9c58ade7bf8c3ca8959b64d8031d36d9b1c62f3f1c51c7cb2031072c7c801b5c1614dae441383a65344acd238f13db28ff0a39c0626e597f002062552d64c616d8b2a6a93d22936055110c0065728aa2b4fbf4d76b108390b474203322d3c93c741674a307cf6455e77c02ceeda307d4ec23fd809a2a420b4243f82052ab92a9cedc6716ad4c66a8a3e423b195b05bdebde456f992bff48f2561e99720e6379995e7053823b8ba8fb8af9623cf48e89f60c989598445df5e711db42a6f20192894ed637e86561ff6a4b8dea4539dee8bddb2fb20bf4ae3499852985c88b120e0005edd09f2335aa6b59ff4723e1262b2192adaa5e3e56f79e662f07041f04c2033577f3e2c5bb0e58746980a07cdfad2f872e2b9a10bcc27b7c678c85576df8420f0f04180d15b6eaa0c43e62380084c75ad773d790700a7120c6c4da1fc51693000fd720100209648e8f10a5ff4c209009b9a09697babbe1b2150d0948c1970a560282a1bfa4720988af8f34859dd8309bffea0b1dff9c8cef0b9b0d6a1852d40786627729ae7be00206ebf4f1b7861bca041cbb8feca75158511ca43a1810d17e1e3017468e8cef0de20cac93064090a7da09f8202c17d1e6cbb9a16eb43afcb032e80719cbf05b3446d2019b76a10b91fb99ec08814e8108e5490b879fb09a190cb2c129dfd98335bd5de000020b1da1198bacacf2adc0d863929d77c285ce3a26e736203d0c0a69a1312255fb2207ee8aa092f49348bd89f9c4bf004b0bee2241a2d0acfe7b3ce08e414b04a5717205b0dda71eac8a4e4cdc6a7b939748c0a78abb54f2547a780e6df67b25530330f000020fc358fb9d1e0d36461e015ac8e35f97072a9f9e750a3c25722a2b1a858fcb82d203c52c9fac6d4694b351390158334a9166bc3478ceb9bea2b0b244915f918239e20d526344a24ff19ee6a9f5c5beb833f4eb6d51191590350e26fa50b138493473f005200000000000000000000002077c404fec0a4265568951dbd096572787d109fab105213f4f292a5f53ce72fca00000020b8d1c7a386eaba83ce83ee0700d4ca9b86e75d147d670ea05123e438231d895000004801250b090a0a010b0f0c0305030c090c05040e02010d0f0f0b0407000f06050d090c02020a0006202af2097cf9d3f42e49f6b3c3dd254e7cbdab3485b029721cbbbf1ad0455a810852000000000000002055170506f4b18bc573a909b51cb21bdd5d303ec511f6cdfb1c6a1ab8d8a1dad020ee774c1b9fe1d8ea8d05823837d959da48af74f384d52f06c42c9d146c5258e300000000000000000072000000204457a6fe530ee953ad1f9caf63daf7f86719c9986df2d0b6917021eb379800f00020406bfc79da4ba6f37452a679d13cca252585d34f7e94a480b047bad9427f233e00000000201ce15a2373d28e0dc5f2000cf308f155d06f72070a29e5af1528c8f05f29d248000000000000004301200601060c0601060e06030605040f0700000000000000000000000000000000072091b83866bbd7450115b462e8d48601af3c3e9a35e7018d2b98a23e107c15c200090307000410a328e800",
"success" : true
}`)
js := []byte(`"Bfn///8SBiQBAQ8D6yfHa4wV24kQ9eXarzY5Bw55VFzysUbkJjrz5FipqkjSAAQEBAQEBAMcbFvhto6QJgYoJs/uzqTrZNrPxpkgNiF5Z/ME98copwPQ4q6ZqLA8S7XUXNCrJNF68vMu8Gx3W8Ooo3qwMomm0gQDiT6zHh/siCZ0c2bfBEymPmRNTiXSAKFIammjmnnBnJYD+CNwgcEzBJqYfnc7RMhr8cPhffKN0281w0M7XLQ9BO4D7W+t3cleDNdiNc6tqWR8jyIP+bolh5QnZIyKXPwGHjsEBAQDcpxkuWYJr6g3ilENTh1sztlZsXZvt6Eedmyy6kI2gQoEKQEGDw8PDw8PA33qzf1Q5ILAwmYxBnM2N80A8JtFHKR7UHhVEqo5nQ0eUgADbChDXdc7hSDZpD9xbhYGuJxVxRWqhsVRTR2dE+18gd4DG5gRFexXofB0aNb6G2kzQUSTD+aWVsfmnKGf4HHivzAEBAQEBAQEBAQEBAQEBARSAAQEA2IMPmRKP0b2BqhMB6IgtfpPeuXKJMdMze7Cr1TeJqbmA1vvqQgR5DN9ew+Zp/nc5SBQbjV5gEq7F/tIipWaQJ1hBAQEBAQEBAQEBAQEBAMCAR4="`)
var p GetProof
var p ProofWithKey
require.NoError(t, json.Unmarshal(js, &p))
require.Equal(t, 8, len(p.Result.Proof))
for i := range p.Result.Proof { // smoke test that every chunk is correctly encoded node
r := io.NewBinReaderFromBuf(p.Result.Proof[i])
require.Equal(t, 6, len(p.Proof))
for i := range p.Proof { // smoke test that every chunk is correctly encoded node
r := io.NewBinReaderFromBuf(p.Proof[i])
var n mpt.NodeObject
n.DecodeBinary(r)
require.NoError(t, r.Err)

View file

@ -905,7 +905,7 @@ func (s *Server) getProof(ps request.Params) (interface{}, *response.Error) {
if err != nil {
return nil, response.ErrInvalidParams
}
key, err := ps.Value(2).GetBytesHex()
key, err := ps.Value(2).GetBytesBase64()
if err != nil {
return nil, response.ErrInvalidParams
}
@ -915,12 +915,9 @@ func (s *Server) getProof(ps request.Params) (interface{}, *response.Error) {
}
skey := makeStorageKey(cs.ID, key)
proof, err := s.chain.GetStateModule().GetStateProof(root, skey)
return &result.GetProof{
Result: result.ProofWithKey{
Key: skey,
Proof: proof,
},
Success: err == nil,
return &result.ProofWithKey{
Key: skey,
Proof: proof,
}, nil
}

View file

@ -1335,20 +1335,19 @@ func testRPCProtocol(t *testing.T, doRPCCall func(string, string, *testing.T) []
r, err := chain.GetStateModule().GetStateRoot(3)
require.NoError(t, err)
rpc := fmt.Sprintf(`{"jsonrpc": "2.0", "id": 1, "method": "getproof", "params": ["%s", "%s", "%x"]}`,
r.Root.StringLE(), testContractHash, []byte("testkey"))
rpc := fmt.Sprintf(`{"jsonrpc": "2.0", "id": 1, "method": "getproof", "params": ["%s", "%s", "%s"]}`,
r.Root.StringLE(), testContractHash, base64.StdEncoding.EncodeToString([]byte("testkey")))
body := doRPCCall(rpc, httpSrv.URL, t)
rawRes := checkErrGetResult(t, body, false)
res := new(result.GetProof)
res := new(result.ProofWithKey)
require.NoError(t, json.Unmarshal(rawRes, res))
require.True(t, res.Success)
h, _ := util.Uint160DecodeStringLE(testContractHash)
skey := makeStorageKey(chain.GetContractState(h).ID, []byte("testkey"))
require.Equal(t, skey, res.Result.Key)
require.True(t, len(res.Result.Proof) > 0)
require.Equal(t, skey, res.Key)
require.True(t, len(res.Proof) > 0)
rpc = fmt.Sprintf(`{"jsonrpc": "2.0", "id": 1, "method": "verifyproof", "params": ["%s", "%s"]}`,
r.Root.StringLE(), res.Result.String())
r.Root.StringLE(), res.String())
body = doRPCCall(rpc, httpSrv.URL, t)
rawRes = checkErrGetResult(t, body, false)
vp := new(result.VerifyProof)