From d04b00074815e5ea385d7cdd2e67e9ba86ca8c50 Mon Sep 17 00:00:00 2001 From: Evgeniy Stratonikov Date: Tue, 12 Jan 2021 11:42:09 +0300 Subject: [PATCH 1/7] vm: remove iterator/enumerator Concat API Follow neo-project/neo#2170. --- pkg/compiler/syscall.go | 2 - pkg/core/interop/enumerator/interop.go | 5 --- pkg/core/interop/enumerator/interop_test.go | 5 +-- pkg/core/interop/interopnames/names.go | 4 -- pkg/core/interop/iterator/interop.go | 5 --- pkg/core/interop/iterator/interop_test.go | 5 +-- pkg/core/interops.go | 2 - pkg/interop/enumerator/enumerator.go | 9 ----- pkg/interop/iterator/iterator.go | 11 ------ pkg/vm/interop.go | 38 ------------------ pkg/vm/interop_iterators.go | 39 ------------------ pkg/vm/vm_test.go | 44 --------------------- 12 files changed, 2 insertions(+), 167 deletions(-) diff --git a/pkg/compiler/syscall.go b/pkg/compiler/syscall.go index c30617ab3..eacbe5c7f 100644 --- a/pkg/compiler/syscall.go +++ b/pkg/compiler/syscall.go @@ -36,13 +36,11 @@ var syscalls = map[string]map[string]string{ "SHA256": interopnames.NeoCryptoSHA256, }, "enumerator": { - "Concat": interopnames.SystemEnumeratorConcat, "Create": interopnames.SystemEnumeratorCreate, "Next": interopnames.SystemEnumeratorNext, "Value": interopnames.SystemEnumeratorValue, }, "iterator": { - "Concat": interopnames.SystemIteratorConcat, "Create": interopnames.SystemIteratorCreate, "Key": interopnames.SystemIteratorKey, "Keys": interopnames.SystemIteratorKeys, diff --git a/pkg/core/interop/enumerator/interop.go b/pkg/core/interop/enumerator/interop.go index 24cc85931..b75daff0c 100644 --- a/pkg/core/interop/enumerator/interop.go +++ b/pkg/core/interop/enumerator/interop.go @@ -5,11 +5,6 @@ import ( "github.com/nspcc-dev/neo-go/pkg/vm" ) -// Concat concatenates 2 enumerators into a single one. -func Concat(ic *interop.Context) error { - return vm.EnumeratorConcat(ic.VM) -} - // Create creates an enumerator from an array-like or bytearray-like stack item. func Create(ic *interop.Context) error { return vm.EnumeratorCreate(ic.VM) diff --git a/pkg/core/interop/enumerator/interop_test.go b/pkg/core/interop/enumerator/interop_test.go index 20149455d..a77a369a3 100644 --- a/pkg/core/interop/enumerator/interop_test.go +++ b/pkg/core/interop/enumerator/interop_test.go @@ -13,11 +13,8 @@ import ( func TestEnumerator(t *testing.T) { ic := &interop.Context{VM: vm.New()} full := []byte{4, 8, 15} - ic.VM.Estack().PushVal(full[2:]) + ic.VM.Estack().PushVal(full) require.NoError(t, Create(ic)) - ic.VM.Estack().PushVal(full[:2]) - require.NoError(t, Create(ic)) - require.NoError(t, Concat(ic)) res := ic.VM.Estack().Pop().Item() for i := range full { diff --git a/pkg/core/interop/interopnames/names.go b/pkg/core/interop/interopnames/names.go index 1d642cc9b..23b3fa3a4 100644 --- a/pkg/core/interop/interopnames/names.go +++ b/pkg/core/interop/interopnames/names.go @@ -26,11 +26,9 @@ const ( SystemContractGetCallFlags = "System.Contract.GetCallFlags" SystemContractNativeOnPersist = "System.Contract.NativeOnPersist" SystemContractNativePostPersist = "System.Contract.NativePostPersist" - SystemEnumeratorConcat = "System.Enumerator.Concat" SystemEnumeratorCreate = "System.Enumerator.Create" SystemEnumeratorNext = "System.Enumerator.Next" SystemEnumeratorValue = "System.Enumerator.Value" - SystemIteratorConcat = "System.Iterator.Concat" SystemIteratorCreate = "System.Iterator.Create" SystemIteratorKey = "System.Iterator.Key" SystemIteratorKeys = "System.Iterator.Keys" @@ -91,11 +89,9 @@ var names = []string{ SystemContractGetCallFlags, SystemContractNativeOnPersist, SystemContractNativePostPersist, - SystemEnumeratorConcat, SystemEnumeratorCreate, SystemEnumeratorNext, SystemEnumeratorValue, - SystemIteratorConcat, SystemIteratorCreate, SystemIteratorKey, SystemIteratorKeys, diff --git a/pkg/core/interop/iterator/interop.go b/pkg/core/interop/iterator/interop.go index 6bc161479..7692a8c2f 100644 --- a/pkg/core/interop/iterator/interop.go +++ b/pkg/core/interop/iterator/interop.go @@ -5,11 +5,6 @@ import ( "github.com/nspcc-dev/neo-go/pkg/vm" ) -// Concat concatenates 2 iterators into a single one. -func Concat(ic *interop.Context) error { - return vm.IteratorConcat(ic.VM) -} - // Create creates an iterator from array-like or map stack item. func Create(ic *interop.Context) error { return vm.IteratorCreate(ic.VM) diff --git a/pkg/core/interop/iterator/interop_test.go b/pkg/core/interop/iterator/interop_test.go index b220b5271..c8324b9ed 100644 --- a/pkg/core/interop/iterator/interop_test.go +++ b/pkg/core/interop/iterator/interop_test.go @@ -13,11 +13,8 @@ import ( func TestIterator(t *testing.T) { ic := &interop.Context{VM: vm.New()} full := []byte{4, 8, 15} - ic.VM.Estack().PushVal(full[2:]) + ic.VM.Estack().PushVal(full) require.NoError(t, Create(ic)) - ic.VM.Estack().PushVal(full[:2]) - require.NoError(t, Create(ic)) - require.NoError(t, Concat(ic)) res := ic.VM.Estack().Pop().Item() ic.VM.Estack().PushVal(res) diff --git a/pkg/core/interops.go b/pkg/core/interops.go index 41cbb1f25..eff9006ac 100644 --- a/pkg/core/interops.go +++ b/pkg/core/interops.go @@ -58,11 +58,9 @@ var systemInterops = []interop.Function{ {Name: interopnames.SystemContractGetCallFlags, Func: contractGetCallFlags, Price: 1 << 10}, {Name: interopnames.SystemContractNativeOnPersist, Func: native.OnPersist, Price: 0, RequiredFlags: callflag.WriteStates}, {Name: interopnames.SystemContractNativePostPersist, Func: native.PostPersist, Price: 0, RequiredFlags: callflag.WriteStates}, - {Name: interopnames.SystemEnumeratorConcat, Func: enumerator.Concat, Price: 1 << 4, ParamCount: 2}, {Name: interopnames.SystemEnumeratorCreate, Func: enumerator.Create, Price: 1 << 4, ParamCount: 1}, {Name: interopnames.SystemEnumeratorNext, Func: enumerator.Next, Price: 1 << 15, ParamCount: 1}, {Name: interopnames.SystemEnumeratorValue, Func: enumerator.Value, Price: 1 << 4, ParamCount: 1}, - {Name: interopnames.SystemIteratorConcat, Func: iterator.Concat, Price: 1 << 4, ParamCount: 2}, {Name: interopnames.SystemIteratorCreate, Func: iterator.Create, Price: 1 << 4, ParamCount: 1}, {Name: interopnames.SystemIteratorKey, Func: iterator.Key, Price: 1 << 4, ParamCount: 1}, {Name: interopnames.SystemIteratorKeys, Func: iterator.Keys, Price: 1 << 4, ParamCount: 1}, diff --git a/pkg/interop/enumerator/enumerator.go b/pkg/interop/enumerator/enumerator.go index 5581538a8..740f270a3 100644 --- a/pkg/interop/enumerator/enumerator.go +++ b/pkg/interop/enumerator/enumerator.go @@ -30,12 +30,3 @@ func Next(e Enumerator) bool { func Value(e Enumerator) interface{} { return nil } - -// Concat concatenates two given enumerators returning one that will range on -// a first and then continue with b. Enumerator positions are not reset for a -// and b, so if any of them was already advanced by Next the resulting -// Enumerator will point at this new position and never go back to previous -// values. This function uses `System.Enumerator.Concat` syscall. -func Concat(a, b Enumerator) Enumerator { - return Enumerator{} -} diff --git a/pkg/interop/iterator/iterator.go b/pkg/interop/iterator/iterator.go index 80be62165..d99729518 100644 --- a/pkg/interop/iterator/iterator.go +++ b/pkg/interop/iterator/iterator.go @@ -19,17 +19,6 @@ func Create(items interface{}) Iterator { return Iterator{} } -// Concat concatenates two given iterators returning one that will range on -// a first and then continue with b. Iterator positions are not reset for a -// and b, so if any of them was already advanced by Next the resulting -// Iterator will point at this new position and never go back to previous -// key-value pairs. Concatenated iterators also remain completely independent -// in results they return, so if both contain the same key you'll receive this -// key twice when iterating. This function uses `System.Iterator.Concat` syscall. -func Concat(a, b Iterator) Iterator { - return Iterator{} -} - // Key returns iterator's key at current position. It's only valid to call after // successful Next call. This function uses `System.Iterator.Key` syscall. func Key(it Iterator) interface{} { diff --git a/pkg/vm/interop.go b/pkg/vm/interop.go index be45f3971..295d5f3cc 100644 --- a/pkg/vm/interop.go +++ b/pkg/vm/interop.go @@ -31,14 +31,10 @@ var defaultVMInterops = []interopIDFuncPrice{ Func: EnumeratorCreate, Price: 1 << 4}, {ID: interopnames.ToID([]byte(interopnames.SystemEnumeratorNext)), Func: EnumeratorNext, Price: 1 << 15}, - {ID: interopnames.ToID([]byte(interopnames.SystemEnumeratorConcat)), - Func: EnumeratorConcat, Price: 1 << 4}, {ID: interopnames.ToID([]byte(interopnames.SystemEnumeratorValue)), Func: EnumeratorValue, Price: 1 << 4}, {ID: interopnames.ToID([]byte(interopnames.SystemIteratorCreate)), Func: IteratorCreate, Price: 1 << 4}, - {ID: interopnames.ToID([]byte(interopnames.SystemIteratorConcat)), - Func: IteratorConcat, Price: 1 << 4}, {ID: interopnames.ToID([]byte(interopnames.SystemIteratorKey)), Func: IteratorKey, Price: 1 << 4}, {ID: interopnames.ToID([]byte(interopnames.SystemIteratorKeys)), @@ -160,23 +156,6 @@ func EnumeratorValue(v *VM) error { return nil } -// EnumeratorConcat handles syscall System.Enumerator.Concat. -func EnumeratorConcat(v *VM) error { - iop1 := v.Estack().Pop().Interop() - arr1 := iop1.Value().(enumerator) - iop2 := v.Estack().Pop().Interop() - arr2 := iop2.Value().(enumerator) - - v.Estack().Push(&Element{ - value: stackitem.NewInterop(&concatEnum{ - current: arr1, - second: arr2, - }), - }) - - return nil -} - // IteratorCreate handles syscall System.Iterator.Create. func IteratorCreate(v *VM) error { data := v.Estack().Pop() @@ -212,23 +191,6 @@ func NewMapIterator(m *stackitem.Map) *stackitem.Interop { }) } -// IteratorConcat handles syscall System.Iterator.Concat. -func IteratorConcat(v *VM) error { - iop1 := v.Estack().Pop().Interop() - iter1 := iop1.Value().(iterator) - iop2 := v.Estack().Pop().Interop() - iter2 := iop2.Value().(iterator) - - v.Estack().Push(&Element{value: stackitem.NewInterop( - &concatIter{ - current: iter1, - second: iter2, - }, - )}) - - return nil -} - // IteratorKey handles syscall System.Iterator.Key. func IteratorKey(v *VM) error { iop := v.estack.Pop().Interop() diff --git a/pkg/vm/interop_iterators.go b/pkg/vm/interop_iterators.go index b29de47e3..d87396c96 100644 --- a/pkg/vm/interop_iterators.go +++ b/pkg/vm/interop_iterators.go @@ -22,10 +22,6 @@ type ( value []byte } - concatEnum struct { - current enumerator - second enumerator - } ) type ( @@ -39,11 +35,6 @@ type ( m []stackitem.MapElement } - concatIter struct { - current iterator - second iterator - } - keysWrapper struct { iter iterator } @@ -87,36 +78,6 @@ func (a *byteArrayWrapper) Key() stackitem.Item { return stackitem.Make(a.index) } -func (c *concatEnum) Next() bool { - if c.current.Next() { - return true - } - c.current = c.second - - return c.current.Next() -} - -func (c *concatEnum) Value() stackitem.Item { - return c.current.Value() -} - -func (i *concatIter) Next() bool { - if i.current.Next() { - return true - } - i.current = i.second - - return i.second.Next() -} - -func (i *concatIter) Value() stackitem.Item { - return i.current.Value() -} - -func (i *concatIter) Key() stackitem.Item { - return i.current.Key() -} - func (m *mapWrapper) Next() bool { if next := m.index + 1; next < len(m.m) { m.index = next diff --git a/pkg/vm/vm_test.go b/pkg/vm/vm_test.go index 7a924c10d..be0c03103 100644 --- a/pkg/vm/vm_test.go +++ b/pkg/vm/vm_test.go @@ -545,50 +545,6 @@ func TestIteratorCreate(t *testing.T) { }) } -func testIterableConcat(t *testing.T, typ string) { - isIter := typ == "Iterator" - prog := getSyscallProg("System." + typ + ".Create") - prog = append(prog, byte(opcode.SWAP)) - prog = append(prog, getSyscallProg("System."+typ+".Create")...) - prog = append(prog, getSyscallProg("System."+typ+".Concat")...) - prog = append(prog, getEnumeratorProg(3, isIter)...) - vm := load(prog) - - arr := []stackitem.Item{ - stackitem.NewBool(false), - stackitem.NewBigInteger(big.NewInt(123)), - stackitem.NewMap(), - } - vm.estack.Push(&Element{value: stackitem.NewArray(arr[:1])}) - vm.estack.Push(&Element{value: stackitem.NewArray(arr[1:])}) - - runVM(t, vm) - - if isIter { - // Yes, this is how iterators are concatenated in reference VM - // https://github.com/neo-project/neo/blob/master-2.x/neo.UnitTests/UT_ConcatenatedIterator.cs#L54 - checkEnumeratorStack(t, vm, []stackitem.Item{ - stackitem.Make(1), arr[2], stackitem.NewBool(true), - stackitem.Make(0), arr[1], stackitem.NewBool(true), - stackitem.Make(0), arr[0], stackitem.NewBool(true), - }) - } else { - checkEnumeratorStack(t, vm, []stackitem.Item{ - arr[2], stackitem.NewBool(true), - arr[1], stackitem.NewBool(true), - arr[0], stackitem.NewBool(true), - }) - } -} - -func TestEnumeratorConcat(t *testing.T) { - testIterableConcat(t, "Enumerator") -} - -func TestIteratorConcat(t *testing.T) { - testIterableConcat(t, "Iterator") -} - func TestIteratorKeys(t *testing.T) { prog := getSyscallProg(interopnames.SystemIteratorCreate) prog = append(prog, getSyscallProg(interopnames.SystemIteratorKeys)...) From 2130e17f0c66adfff60f2d08707c3f7212cf9a1d Mon Sep 17 00:00:00 2001 From: Evgeniy Stratonikov Date: Tue, 12 Jan 2021 12:30:21 +0300 Subject: [PATCH 2/7] core,vm: remove `System.Enumerator.*` interops Map iterator now returns key-value pair, while array/byte-array iterators work like old enumerators. Follow neo-project/neo#2190. --- examples/iterator/iterator.go | 11 +-- examples/iterator/iterator.yml | 8 +- examples/storage/storage.go | 5 +- pkg/compiler/syscall.go | 12 +-- pkg/core/interop/enumerator/interop.go | 22 ----- pkg/core/interop/enumerator/interop_test.go | 32 ------- pkg/core/interop/interopnames/names.go | 16 +--- pkg/core/interop/iterator/interop.go | 19 ++--- pkg/core/interop/iterator/interop_test.go | 36 +++----- pkg/core/interop_neo_test.go | 45 ++++------ pkg/core/interops.go | 9 +- pkg/interop/enumerator/enumerator.go | 32 ------- pkg/interop/iterator/iterator.go | 30 +------ pkg/vm/interop.go | 85 +++--------------- pkg/vm/interop_iterators.go | 51 ++--------- pkg/vm/vm_test.go | 95 +++------------------ 16 files changed, 86 insertions(+), 422 deletions(-) delete mode 100644 pkg/core/interop/enumerator/interop.go delete mode 100644 pkg/core/interop/enumerator/interop_test.go delete mode 100644 pkg/interop/enumerator/enumerator.go diff --git a/examples/iterator/iterator.go b/examples/iterator/iterator.go index 534bd0885..5f91204a0 100644 --- a/examples/iterator/iterator.go +++ b/examples/iterator/iterator.go @@ -9,13 +9,8 @@ import ( // NotifyKeysAndValues sends notification with `foo` storage keys and values func NotifyKeysAndValues() bool { iter := storage.Find(storage.GetContext(), []byte("foo")) - values := iterator.Values(iter) - keys := iterator.Keys(iter) - - runtime.Notify("found storage values", values) - // For illustration purposes event is emitted with 'Any' type. - var typedKeys interface{} = keys - runtime.Notify("found storage keys", typedKeys) - + for iterator.Next(iter) { + runtime.Notify("found storage key-value pair", iterator.Value(iter)) + } return true } diff --git a/examples/iterator/iterator.yml b/examples/iterator/iterator.yml index e714e9d8a..3f881c477 100644 --- a/examples/iterator/iterator.yml +++ b/examples/iterator/iterator.yml @@ -1,11 +1,7 @@ name: "Iterator example" supportedstandards: [] events: - - name: found storage values + - name: found storage key-value pair parameters: - - name: values - type: InteropInterface - - name: found storage keys - parameters: - - name: keys + - name: value type: Any diff --git a/examples/storage/storage.go b/examples/storage/storage.go index 9b43c45b6..963dfa2a4 100644 --- a/examples/storage/storage.go +++ b/examples/storage/storage.go @@ -35,9 +35,8 @@ func Find(value []byte) []string { iter := storage.Find(ctx, value) result := []string{} for iterator.Next(iter) { - val := iterator.Value(iter) - key := iterator.Key(iter) - result = append(result, key.(string)+":"+val.(string)) + val := iterator.Value(iter).([]string) + result = append(result, val[0]+":"+val[1]) } return result } diff --git a/pkg/compiler/syscall.go b/pkg/compiler/syscall.go index eacbe5c7f..ead24c899 100644 --- a/pkg/compiler/syscall.go +++ b/pkg/compiler/syscall.go @@ -35,18 +35,10 @@ var syscalls = map[string]map[string]string{ "RIPEMD160": interopnames.NeoCryptoRIPEMD160, "SHA256": interopnames.NeoCryptoSHA256, }, - "enumerator": { - "Create": interopnames.SystemEnumeratorCreate, - "Next": interopnames.SystemEnumeratorNext, - "Value": interopnames.SystemEnumeratorValue, - }, "iterator": { "Create": interopnames.SystemIteratorCreate, - "Key": interopnames.SystemIteratorKey, - "Keys": interopnames.SystemIteratorKeys, - "Next": interopnames.SystemEnumeratorNext, - "Value": interopnames.SystemEnumeratorValue, - "Values": interopnames.SystemIteratorValues, + "Next": interopnames.SystemIteratorNext, + "Value": interopnames.SystemIteratorValue, }, "json": { "Deserialize": interopnames.SystemJSONDeserialize, diff --git a/pkg/core/interop/enumerator/interop.go b/pkg/core/interop/enumerator/interop.go deleted file mode 100644 index b75daff0c..000000000 --- a/pkg/core/interop/enumerator/interop.go +++ /dev/null @@ -1,22 +0,0 @@ -package enumerator - -import ( - "github.com/nspcc-dev/neo-go/pkg/core/interop" - "github.com/nspcc-dev/neo-go/pkg/vm" -) - -// Create creates an enumerator from an array-like or bytearray-like stack item. -func Create(ic *interop.Context) error { - return vm.EnumeratorCreate(ic.VM) -} - -// Next advances the enumerator, pushes true if is it was successful -// and false otherwise. -func Next(ic *interop.Context) error { - return vm.EnumeratorNext(ic.VM) -} - -// Value returns the current value of the enumerator. -func Value(ic *interop.Context) error { - return vm.EnumeratorValue(ic.VM) -} diff --git a/pkg/core/interop/enumerator/interop_test.go b/pkg/core/interop/enumerator/interop_test.go deleted file mode 100644 index a77a369a3..000000000 --- a/pkg/core/interop/enumerator/interop_test.go +++ /dev/null @@ -1,32 +0,0 @@ -package enumerator - -import ( - "math/big" - "testing" - - "github.com/nspcc-dev/neo-go/pkg/core/interop" - "github.com/nspcc-dev/neo-go/pkg/vm" - "github.com/stretchr/testify/require" -) - -// Enumerator is thoroughly tested in VM package, these are smoke tests. -func TestEnumerator(t *testing.T) { - ic := &interop.Context{VM: vm.New()} - full := []byte{4, 8, 15} - ic.VM.Estack().PushVal(full) - require.NoError(t, Create(ic)) - - res := ic.VM.Estack().Pop().Item() - for i := range full { - ic.VM.Estack().PushVal(res) - require.NoError(t, Next(ic)) - require.True(t, ic.VM.Estack().Pop().Bool()) - ic.VM.Estack().PushVal(res) - require.NoError(t, Value(ic)) - require.Equal(t, big.NewInt(int64(full[i])), ic.VM.Estack().Pop().BigInt()) - } - - ic.VM.Estack().PushVal(res) - require.NoError(t, Next(ic)) - require.False(t, ic.VM.Estack().Pop().Bool()) -} diff --git a/pkg/core/interop/interopnames/names.go b/pkg/core/interop/interopnames/names.go index 23b3fa3a4..8a76df3dd 100644 --- a/pkg/core/interop/interopnames/names.go +++ b/pkg/core/interop/interopnames/names.go @@ -26,13 +26,9 @@ const ( SystemContractGetCallFlags = "System.Contract.GetCallFlags" SystemContractNativeOnPersist = "System.Contract.NativeOnPersist" SystemContractNativePostPersist = "System.Contract.NativePostPersist" - SystemEnumeratorCreate = "System.Enumerator.Create" - SystemEnumeratorNext = "System.Enumerator.Next" - SystemEnumeratorValue = "System.Enumerator.Value" SystemIteratorCreate = "System.Iterator.Create" - SystemIteratorKey = "System.Iterator.Key" - SystemIteratorKeys = "System.Iterator.Keys" - SystemIteratorValues = "System.Iterator.Values" + SystemIteratorNext = "System.Iterator.Next" + SystemIteratorValue = "System.Iterator.Value" SystemJSONDeserialize = "System.Json.Deserialize" SystemJSONSerialize = "System.Json.Serialize" SystemRuntimeCheckWitness = "System.Runtime.CheckWitness" @@ -89,13 +85,9 @@ var names = []string{ SystemContractGetCallFlags, SystemContractNativeOnPersist, SystemContractNativePostPersist, - SystemEnumeratorCreate, - SystemEnumeratorNext, - SystemEnumeratorValue, SystemIteratorCreate, - SystemIteratorKey, - SystemIteratorKeys, - SystemIteratorValues, + SystemIteratorNext, + SystemIteratorValue, SystemJSONDeserialize, SystemJSONSerialize, SystemRuntimeCheckWitness, diff --git a/pkg/core/interop/iterator/interop.go b/pkg/core/interop/iterator/interop.go index 7692a8c2f..f9efa9d09 100644 --- a/pkg/core/interop/iterator/interop.go +++ b/pkg/core/interop/iterator/interop.go @@ -10,17 +10,14 @@ func Create(ic *interop.Context) error { return vm.IteratorCreate(ic.VM) } -// Key returns current iterator key. -func Key(ic *interop.Context) error { - return vm.IteratorKey(ic.VM) +// Next advances the iterator, pushes true on success and false otherwise. +func Next(ic *interop.Context) error { + return vm.IteratorNext(ic.VM) } -// Keys returns keys of the iterator. -func Keys(ic *interop.Context) error { - return vm.IteratorKeys(ic.VM) -} - -// Values returns values of the iterator. -func Values(ic *interop.Context) error { - return vm.IteratorValues(ic.VM) +// Value returns current iterator value and depends on iterator type: +// For slices the result is just value. +// For maps the result is key-value pair packed in a struct. +func Value(ic *interop.Context) error { + return vm.IteratorValue(ic.VM) } diff --git a/pkg/core/interop/iterator/interop_test.go b/pkg/core/interop/iterator/interop_test.go index c8324b9ed..fe830fea8 100644 --- a/pkg/core/interop/iterator/interop_test.go +++ b/pkg/core/interop/iterator/interop_test.go @@ -17,29 +17,19 @@ func TestIterator(t *testing.T) { require.NoError(t, Create(ic)) res := ic.VM.Estack().Pop().Item() - ic.VM.Estack().PushVal(res) - require.NoError(t, vm.EnumeratorNext(ic.VM)) - require.True(t, ic.VM.Estack().Pop().Bool()) + for i := range full { + ic.VM.Estack().PushVal(res) + require.NoError(t, Next(ic)) + require.True(t, ic.VM.Estack().Pop().Bool()) + + ic.VM.Estack().PushVal(res) + require.NoError(t, Value(ic)) + + value := ic.VM.Estack().Pop().Item().Value() + require.Equal(t, big.NewInt(int64(full[i])), value) + } ic.VM.Estack().PushVal(res) - require.NoError(t, Key(ic)) - require.Equal(t, big.NewInt(0), ic.VM.Estack().Pop().BigInt()) - - ic.VM.Estack().PushVal(res) - require.NoError(t, vm.EnumeratorValue(ic.VM)) - require.Equal(t, big.NewInt(int64(full[0])), ic.VM.Estack().Pop().BigInt()) - - ic.VM.Estack().PushVal(res) - require.NoError(t, vm.EnumeratorNext(ic.VM)) - require.True(t, ic.VM.Estack().Pop().Bool()) - - ic.VM.Estack().PushVal(res) - require.NoError(t, Keys(ic)) - require.NoError(t, vm.EnumeratorValue(ic.VM)) - require.Equal(t, big.NewInt(1), ic.VM.Estack().Pop().BigInt()) - - ic.VM.Estack().PushVal(res) - require.NoError(t, Values(ic)) - require.NoError(t, vm.EnumeratorValue(ic.VM)) - require.Equal(t, big.NewInt(int64(full[1])), ic.VM.Estack().Pop().BigInt()) + require.NoError(t, Next(ic)) + require.False(t, false, ic.VM.Estack().Pop().Bool()) } diff --git a/pkg/core/interop_neo_test.go b/pkg/core/interop_neo_test.go index 53fd9de61..8c2bdf7db 100644 --- a/pkg/core/interop_neo_test.go +++ b/pkg/core/interop_neo_test.go @@ -7,7 +7,6 @@ import ( "github.com/nspcc-dev/neo-go/pkg/core/block" "github.com/nspcc-dev/neo-go/pkg/core/dao" "github.com/nspcc-dev/neo-go/pkg/core/interop" - "github.com/nspcc-dev/neo-go/pkg/core/interop/enumerator" "github.com/nspcc-dev/neo-go/pkg/core/interop/iterator" "github.com/nspcc-dev/neo-go/pkg/core/state" "github.com/nspcc-dev/neo-go/pkg/core/storage" @@ -67,33 +66,25 @@ func TestStorageFind(t *testing.T) { require.NoError(t, err) var iter *stackitem.Interop - require.NotPanics(t, func() { iter = v.Estack().Top().Interop() }) + require.NotPanics(t, func() { iter = v.Estack().Pop().Interop() }) - require.NoError(t, enumerator.Next(context)) - require.True(t, v.Estack().Pop().Bool()) + for _, id := range []int{2, 0} { // sorted indices with mathing prefix + v.Estack().PushVal(iter) + require.NoError(t, iterator.Next(context)) + require.True(t, v.Estack().Pop().Bool()) + + v.Estack().PushVal(iter) + require.NoError(t, iterator.Value(context)) + + kv, ok := v.Estack().Pop().Value().([]stackitem.Item) + require.True(t, ok) + require.Len(t, kv, 2) + require.Equal(t, skeys[id], kv[0].Value()) + require.Equal(t, items[id].Value, kv[1].Value()) + } v.Estack().PushVal(iter) - require.NoError(t, iterator.Key(context)) - require.Equal(t, []byte{0x01, 0x01}, v.Estack().Pop().Bytes()) - - v.Estack().PushVal(iter) - require.NoError(t, enumerator.Value(context)) - require.Equal(t, []byte{0x03, 0x04, 0x05, 0x06}, v.Estack().Pop().Bytes()) - - v.Estack().PushVal(iter) - require.NoError(t, enumerator.Next(context)) - require.True(t, v.Estack().Pop().Bool()) - - v.Estack().PushVal(iter) - require.NoError(t, iterator.Key(context)) - require.Equal(t, []byte{0x01, 0x02}, v.Estack().Pop().Bytes()) - - v.Estack().PushVal(iter) - require.NoError(t, enumerator.Value(context)) - require.Equal(t, []byte{0x01, 0x02, 0x03, 0x04}, v.Estack().Pop().Bytes()) - - v.Estack().PushVal(iter) - require.NoError(t, enumerator.Next(context)) + require.NoError(t, iterator.Next(context)) require.False(t, v.Estack().Pop().Bool()) }) @@ -104,7 +95,7 @@ func TestStorageFind(t *testing.T) { err := storageFind(context) require.NoError(t, err) - require.NoError(t, enumerator.Next(context)) + require.NoError(t, iterator.Next(context)) require.False(t, v.Estack().Pop().Bool()) }) @@ -122,7 +113,7 @@ func TestStorageFind(t *testing.T) { v.Estack().PushVal(stackitem.NewInterop(&StorageContext{ID: invalidID})) require.NoError(t, storageFind(context)) - require.NoError(t, enumerator.Next(context)) + require.NoError(t, iterator.Next(context)) require.False(t, v.Estack().Pop().Bool()) }) } diff --git a/pkg/core/interops.go b/pkg/core/interops.go index eff9006ac..08dccd64d 100644 --- a/pkg/core/interops.go +++ b/pkg/core/interops.go @@ -12,7 +12,6 @@ import ( "github.com/nspcc-dev/neo-go/pkg/core/interop/binary" "github.com/nspcc-dev/neo-go/pkg/core/interop/contract" "github.com/nspcc-dev/neo-go/pkg/core/interop/crypto" - "github.com/nspcc-dev/neo-go/pkg/core/interop/enumerator" "github.com/nspcc-dev/neo-go/pkg/core/interop/interopnames" "github.com/nspcc-dev/neo-go/pkg/core/interop/iterator" "github.com/nspcc-dev/neo-go/pkg/core/interop/json" @@ -58,13 +57,9 @@ var systemInterops = []interop.Function{ {Name: interopnames.SystemContractGetCallFlags, Func: contractGetCallFlags, Price: 1 << 10}, {Name: interopnames.SystemContractNativeOnPersist, Func: native.OnPersist, Price: 0, RequiredFlags: callflag.WriteStates}, {Name: interopnames.SystemContractNativePostPersist, Func: native.PostPersist, Price: 0, RequiredFlags: callflag.WriteStates}, - {Name: interopnames.SystemEnumeratorCreate, Func: enumerator.Create, Price: 1 << 4, ParamCount: 1}, - {Name: interopnames.SystemEnumeratorNext, Func: enumerator.Next, Price: 1 << 15, ParamCount: 1}, - {Name: interopnames.SystemEnumeratorValue, Func: enumerator.Value, Price: 1 << 4, ParamCount: 1}, {Name: interopnames.SystemIteratorCreate, Func: iterator.Create, Price: 1 << 4, ParamCount: 1}, - {Name: interopnames.SystemIteratorKey, Func: iterator.Key, Price: 1 << 4, ParamCount: 1}, - {Name: interopnames.SystemIteratorKeys, Func: iterator.Keys, Price: 1 << 4, ParamCount: 1}, - {Name: interopnames.SystemIteratorValues, Func: iterator.Values, Price: 1 << 4, ParamCount: 1}, + {Name: interopnames.SystemIteratorNext, Func: iterator.Next, Price: 1 << 15, ParamCount: 1}, + {Name: interopnames.SystemIteratorValue, Func: iterator.Value, Price: 1 << 4, ParamCount: 1}, {Name: interopnames.SystemJSONDeserialize, Func: json.Deserialize, Price: 1 << 14, ParamCount: 1}, {Name: interopnames.SystemJSONSerialize, Func: json.Serialize, Price: 1 << 12, ParamCount: 1}, {Name: interopnames.SystemRuntimeCheckWitness, Func: runtime.CheckWitness, Price: 1 << 10, diff --git a/pkg/interop/enumerator/enumerator.go b/pkg/interop/enumerator/enumerator.go deleted file mode 100644 index 740f270a3..000000000 --- a/pkg/interop/enumerator/enumerator.go +++ /dev/null @@ -1,32 +0,0 @@ -/* -Package enumerator provides functions to work with enumerators. -*/ -package enumerator - -// Enumerator represents NEO enumerator type, it's an opaque data structure -// that can be used with functions from this package. It's similar to more -// widely used Iterator (see `iterator` package), but ranging over arrays -// or structures that have values with no explicit keys. -type Enumerator struct{} - -// Create creates a new enumerator from the given items (slice, structure, byte -// array and integer or boolean converted to byte array). New enumerator points -// at index -1 of its items, so the user of it has to advance it first with Next. -// This function uses `System.Enumerator.Create` syscall. -func Create(items interface{}) Enumerator { - return Enumerator{} -} - -// Next moves position of the given enumerator by one and returns a bool that -// tells whether there is a new value present in this new position. If it is, -// you can use Value to get it, if not then there are no more values in this -// enumerator. This function uses `System.Enumerator.Next` syscall. -func Next(e Enumerator) bool { - return true -} - -// Value returns current enumerator's item value, it's only valid to call it -// after Next returning true. This function uses `System.Enumerator.Value` syscall. -func Value(e Enumerator) interface{} { - return nil -} diff --git a/pkg/interop/iterator/iterator.go b/pkg/interop/iterator/iterator.go index d99729518..ff124bb84 100644 --- a/pkg/interop/iterator/iterator.go +++ b/pkg/interop/iterator/iterator.go @@ -3,8 +3,6 @@ Package iterator provides functions to work with Neo iterators. */ package iterator -import "github.com/nspcc-dev/neo-go/pkg/interop/enumerator" - // Iterator represents a Neo iterator, it's an opaque data structure that can // be properly created by Create or storage.Find. Unlike enumerators, iterators // range over key-value pairs, so it's convenient to use them for maps. This @@ -19,37 +17,17 @@ func Create(items interface{}) Iterator { return Iterator{} } -// Key returns iterator's key at current position. It's only valid to call after -// successful Next call. This function uses `System.Iterator.Key` syscall. -func Key(it Iterator) interface{} { - return nil -} - -// Keys returns Enumerator ranging over keys or the given Iterator. Note that -// this Enumerator is actually directly tied to the underlying Iterator, so that -// advancing it with Next will actually advance the Iterator too. This function -// uses `System.Iterator.Keys` syscall. -func Keys(it Iterator) enumerator.Enumerator { - return enumerator.Enumerator{} -} - // Next advances the iterator returning true if it is was successful (and you // can use Key or Value) and false otherwise (and there are no more elements in -// this Iterator). This function uses `System.Enumerator.Next` syscall. +// this Iterator). This function uses `System.Iterator.Next` syscall. func Next(it Iterator) bool { return true } // Value returns iterator's current value. It's only valid to call after -// successful Next call. This function uses `System.Enumerator.Value` syscall. +// successful Next call. This function uses `System.Iterator.Value` syscall. +// For slices the result is just value. +// For maps the result can be casted to a slice of 2 elements: key and value. func Value(it Iterator) interface{} { return nil } - -// Values returns Enumerator ranging over values or the given Iterator. Note that -// this Enumerator is actually directly tied to the underlying Iterator, so that -// advancing it with Next will actually advance the Iterator too. This function -// uses `System.Iterator.Values` syscall. -func Values(it Iterator) enumerator.Enumerator { - return enumerator.Enumerator{} -} diff --git a/pkg/vm/interop.go b/pkg/vm/interop.go index 295d5f3cc..cd4f65d17 100644 --- a/pkg/vm/interop.go +++ b/pkg/vm/interop.go @@ -27,20 +27,12 @@ var defaultVMInterops = []interopIDFuncPrice{ Func: runtimeLog, Price: 1 << 15, RequiredFlags: callflag.AllowNotify}, {ID: interopnames.ToID([]byte(interopnames.SystemRuntimeNotify)), Func: runtimeNotify, Price: 1 << 15, RequiredFlags: callflag.AllowNotify}, - {ID: interopnames.ToID([]byte(interopnames.SystemEnumeratorCreate)), - Func: EnumeratorCreate, Price: 1 << 4}, - {ID: interopnames.ToID([]byte(interopnames.SystemEnumeratorNext)), - Func: EnumeratorNext, Price: 1 << 15}, - {ID: interopnames.ToID([]byte(interopnames.SystemEnumeratorValue)), - Func: EnumeratorValue, Price: 1 << 4}, {ID: interopnames.ToID([]byte(interopnames.SystemIteratorCreate)), Func: IteratorCreate, Price: 1 << 4}, - {ID: interopnames.ToID([]byte(interopnames.SystemIteratorKey)), - Func: IteratorKey, Price: 1 << 4}, - {ID: interopnames.ToID([]byte(interopnames.SystemIteratorKeys)), - Func: IteratorKeys, Price: 1 << 4}, - {ID: interopnames.ToID([]byte(interopnames.SystemIteratorValues)), - Func: IteratorValues, Price: 1 << 4}, + {ID: interopnames.ToID([]byte(interopnames.SystemIteratorNext)), + Func: IteratorNext, Price: 1 << 15}, + {ID: interopnames.ToID([]byte(interopnames.SystemIteratorValue)), + Func: IteratorValue, Price: 1 << 4}, } func init() { @@ -112,45 +104,19 @@ func init() { }) } -// EnumeratorCreate handles syscall System.Enumerator.Create. -func EnumeratorCreate(v *VM) error { - var interop interface{} - switch t := v.Estack().Pop().value.(type) { - case *stackitem.Array, *stackitem.Struct: - interop = &arrayWrapper{ - index: -1, - value: t.Value().([]stackitem.Item), - } - default: - data, err := t.TryBytes() - if err != nil { - return fmt.Errorf("can not create enumerator from type %s: %w", t.Type(), err) - } - interop = &byteArrayWrapper{ - index: -1, - value: data, - } - } - v.Estack().Push(&Element{ - value: stackitem.NewInterop(interop), - }) - - return nil -} - -// EnumeratorNext handles syscall System.Enumerator.Next. -func EnumeratorNext(v *VM) error { +// IteratorNext handles syscall System.Enumerator.Next. +func IteratorNext(v *VM) error { iop := v.Estack().Pop().Interop() - arr := iop.Value().(enumerator) + arr := iop.Value().(iterator) v.Estack().PushVal(arr.Next()) return nil } -// EnumeratorValue handles syscall System.Enumerator.Value. -func EnumeratorValue(v *VM) error { +// IteratorValue handles syscall System.Enumerator.Value. +func IteratorValue(v *VM) error { iop := v.Estack().Pop().Interop() - arr := iop.Value().(enumerator) + arr := iop.Value().(iterator) v.Estack().Push(&Element{value: arr.Value()}) return nil @@ -190,34 +156,3 @@ func NewMapIterator(m *stackitem.Map) *stackitem.Interop { m: m.Value().([]stackitem.MapElement), }) } - -// IteratorKey handles syscall System.Iterator.Key. -func IteratorKey(v *VM) error { - iop := v.estack.Pop().Interop() - iter := iop.Value().(iterator) - v.Estack().Push(&Element{value: iter.Key()}) - - return nil -} - -// IteratorKeys handles syscall System.Iterator.Keys. -func IteratorKeys(v *VM) error { - iop := v.estack.Pop().Interop() - iter := iop.Value().(iterator) - v.Estack().Push(&Element{value: stackitem.NewInterop( - &keysWrapper{iter}, - )}) - - return nil -} - -// IteratorValues handles syscall System.Iterator.Values. -func IteratorValues(v *VM) error { - iop := v.estack.Pop().Interop() - iter := iop.Value().(iterator) - v.Estack().Push(&Element{value: stackitem.NewInterop( - &valuesWrapper{iter}, - )}) - - return nil -} diff --git a/pkg/vm/interop_iterators.go b/pkg/vm/interop_iterators.go index d87396c96..2fc07b5f7 100644 --- a/pkg/vm/interop_iterators.go +++ b/pkg/vm/interop_iterators.go @@ -7,7 +7,7 @@ import ( ) type ( - enumerator interface { + iterator interface { Next() bool Value() stackitem.Item } @@ -22,26 +22,10 @@ type ( value []byte } -) - -type ( - iterator interface { - enumerator - Key() stackitem.Item - } - mapWrapper struct { index int m []stackitem.MapElement } - - keysWrapper struct { - iter iterator - } - - valuesWrapper struct { - iter iterator - } ) func (a *arrayWrapper) Next() bool { @@ -57,10 +41,6 @@ func (a *arrayWrapper) Value() stackitem.Item { return a.value[a.index] } -func (a *arrayWrapper) Key() stackitem.Item { - return stackitem.Make(a.index) -} - func (a *byteArrayWrapper) Next() bool { if next := a.index + 1; next < len(a.value) { a.index = next @@ -74,10 +54,6 @@ func (a *byteArrayWrapper) Value() stackitem.Item { return stackitem.NewBigInteger(big.NewInt(int64(a.value[a.index]))) } -func (a *byteArrayWrapper) Key() stackitem.Item { - return stackitem.Make(a.index) -} - func (m *mapWrapper) Next() bool { if next := m.index + 1; next < len(m.m) { m.index = next @@ -88,25 +64,8 @@ func (m *mapWrapper) Next() bool { } func (m *mapWrapper) Value() stackitem.Item { - return m.m[m.index].Value -} - -func (m *mapWrapper) Key() stackitem.Item { - return m.m[m.index].Key -} - -func (e *keysWrapper) Next() bool { - return e.iter.Next() -} - -func (e *keysWrapper) Value() stackitem.Item { - return e.iter.Key() -} - -func (e *valuesWrapper) Next() bool { - return e.iter.Next() -} - -func (e *valuesWrapper) Value() stackitem.Item { - return e.iter.Value() + return stackitem.NewStruct([]stackitem.Item{ + m.m[m.index].Key, + m.m[m.index].Value, + }) } diff --git a/pkg/vm/vm_test.go b/pkg/vm/vm_test.go index be0c03103..4855d28ea 100644 --- a/pkg/vm/vm_test.go +++ b/pkg/vm/vm_test.go @@ -468,20 +468,16 @@ func TestPushData4BigN(t *testing.T) { checkVMFailed(t, vm) } -func getEnumeratorProg(n int, isIter bool) (prog []byte) { +func getIteratorProg(n int) (prog []byte) { prog = []byte{byte(opcode.INITSSLOT), 1, byte(opcode.STSFLD0)} for i := 0; i < n; i++ { prog = append(prog, byte(opcode.LDSFLD0)) - prog = append(prog, getSyscallProg(interopnames.SystemEnumeratorNext)...) + prog = append(prog, getSyscallProg(interopnames.SystemIteratorNext)...) prog = append(prog, byte(opcode.LDSFLD0)) - prog = append(prog, getSyscallProg(interopnames.SystemEnumeratorValue)...) - if isIter { - prog = append(prog, byte(opcode.LDSFLD0)) - prog = append(prog, getSyscallProg(interopnames.SystemIteratorKey)...) - } + prog = append(prog, getSyscallProg(interopnames.SystemIteratorValue)...) } prog = append(prog, byte(opcode.LDSFLD0)) - prog = append(prog, getSyscallProg(interopnames.SystemEnumeratorNext)...) + prog = append(prog, getSyscallProg(interopnames.SystemIteratorNext)...) return } @@ -494,10 +490,9 @@ func checkEnumeratorStack(t *testing.T, vm *VM, arr []stackitem.Item) { } } -func testIterableCreate(t *testing.T, typ string, isByteArray bool) { - isIter := typ == "Iterator" - prog := getSyscallProg("System." + typ + ".Create") - prog = append(prog, getEnumeratorProg(2, isIter)...) +func testIterableCreate(t *testing.T, isByteArray bool) { + prog := getSyscallProg(interopnames.SystemIteratorCreate) + prog = append(prog, getIteratorProg(2)...) vm := load(prog) arr := []stackitem.Item{ @@ -512,32 +507,16 @@ func testIterableCreate(t *testing.T, typ string, isByteArray bool) { } runVM(t, vm) - if isIter { - checkEnumeratorStack(t, vm, []stackitem.Item{ - stackitem.Make(1), arr[1], stackitem.NewBool(true), - stackitem.Make(0), arr[0], stackitem.NewBool(true), - }) - } else { - checkEnumeratorStack(t, vm, []stackitem.Item{ - arr[1], stackitem.NewBool(true), - arr[0], stackitem.NewBool(true), - }) - } -} - -func TestEnumeratorCreate(t *testing.T) { - t.Run("Array", func(t *testing.T) { testIterableCreate(t, "Enumerator", false) }) - t.Run("ByteArray", func(t *testing.T) { testIterableCreate(t, "Enumerator", true) }) - t.Run("Interop", func(t *testing.T) { - v := New() - v.Estack().PushVal(stackitem.NewInterop([]byte{42})) - require.Error(t, EnumeratorCreate(v)) + checkEnumeratorStack(t, vm, []stackitem.Item{ + arr[1], stackitem.NewBool(true), + arr[0], stackitem.NewBool(true), }) } func TestIteratorCreate(t *testing.T) { - t.Run("Array", func(t *testing.T) { testIterableCreate(t, "Iterator", false) }) - t.Run("ByteArray", func(t *testing.T) { testIterableCreate(t, "Iterator", true) }) + t.Run("Array", func(t *testing.T) { testIterableCreate(t, false) }) + t.Run("ByteArray", func(t *testing.T) { testIterableCreate(t, true) }) + t.Run("Map", func(f *testing.T) {}) t.Run("Interop", func(t *testing.T) { v := New() v.Estack().PushVal(stackitem.NewInterop([]byte{42})) @@ -545,54 +524,6 @@ func TestIteratorCreate(t *testing.T) { }) } -func TestIteratorKeys(t *testing.T) { - prog := getSyscallProg(interopnames.SystemIteratorCreate) - prog = append(prog, getSyscallProg(interopnames.SystemIteratorKeys)...) - prog = append(prog, getEnumeratorProg(2, false)...) - - v := load(prog) - arr := stackitem.NewArray([]stackitem.Item{ - stackitem.NewBool(false), - stackitem.NewBigInteger(big.NewInt(42)), - }) - v.estack.PushVal(arr) - - runVM(t, v) - - checkEnumeratorStack(t, v, []stackitem.Item{ - stackitem.NewBigInteger(big.NewInt(1)), stackitem.NewBool(true), - stackitem.NewBigInteger(big.NewInt(0)), stackitem.NewBool(true), - }) -} - -func TestIteratorValues(t *testing.T) { - prog := getSyscallProg(interopnames.SystemIteratorCreate) - prog = append(prog, getSyscallProg(interopnames.SystemIteratorValues)...) - prog = append(prog, getEnumeratorProg(2, false)...) - - v := load(prog) - m := stackitem.NewMap() - m.Add(stackitem.NewBigInteger(big.NewInt(1)), stackitem.NewBool(false)) - m.Add(stackitem.NewByteArray([]byte{32}), stackitem.NewByteArray([]byte{7})) - v.estack.PushVal(m) - - runVM(t, v) - require.Equal(t, 5, v.estack.Len()) - require.Equal(t, stackitem.NewBool(false), v.estack.Peek(0).value) - - // Map values can be enumerated in any order. - i1, i2 := 1, 3 - if _, ok := v.estack.Peek(i1).value.(*stackitem.Bool); !ok { - i1, i2 = i2, i1 - } - - require.Equal(t, stackitem.NewBool(false), v.estack.Peek(i1).value) - require.Equal(t, stackitem.NewByteArray([]byte{7}), v.estack.Peek(i2).value) - - require.Equal(t, stackitem.NewBool(true), v.estack.Peek(2).value) - require.Equal(t, stackitem.NewBool(true), v.estack.Peek(4).value) -} - func getSyscallProg(name string) (prog []byte) { buf := io.NewBufBinWriter() emit.Syscall(buf.BinWriter, name) From 7fc0c04dba00ee38c0b85644095e88b446b3eae7 Mon Sep 17 00:00:00 2001 From: Evgeniy Stratonikov Date: Tue, 12 Jan 2021 13:39:31 +0300 Subject: [PATCH 3/7] core: add flags to `Storage.Find` It can be iterated over keys, values or both. Prefix can be stripped. --- examples/iterator/iterator.go | 2 +- examples/storage/storage.go | 2 +- pkg/compiler/syscall_test.go | 9 ++++ pkg/core/interop/storage/find.go | 51 +++++++++++++++++++++++ pkg/core/interop_neo.go | 19 +++++++-- pkg/core/interop_neo_test.go | 70 ++++++++++++++++++++++++-------- pkg/interop/iterator/iterator.go | 1 + pkg/interop/storage/storage.go | 18 +++++++- 8 files changed, 148 insertions(+), 24 deletions(-) create mode 100644 pkg/core/interop/storage/find.go diff --git a/examples/iterator/iterator.go b/examples/iterator/iterator.go index 5f91204a0..03ffccee9 100644 --- a/examples/iterator/iterator.go +++ b/examples/iterator/iterator.go @@ -8,7 +8,7 @@ import ( // NotifyKeysAndValues sends notification with `foo` storage keys and values func NotifyKeysAndValues() bool { - iter := storage.Find(storage.GetContext(), []byte("foo")) + iter := storage.Find(storage.GetContext(), []byte("foo"), storage.None) for iterator.Next(iter) { runtime.Notify("found storage key-value pair", iterator.Value(iter)) } diff --git a/examples/storage/storage.go b/examples/storage/storage.go index 963dfa2a4..3831ff119 100644 --- a/examples/storage/storage.go +++ b/examples/storage/storage.go @@ -32,7 +32,7 @@ func Delete(key []byte) bool { // Find returns an array of key-value pairs with key that matched the passed value func Find(value []byte) []string { - iter := storage.Find(ctx, value) + iter := storage.Find(ctx, value, storage.None) result := []string{} for iterator.Next(iter) { val := iterator.Value(iter).([]string) diff --git a/pkg/compiler/syscall_test.go b/pkg/compiler/syscall_test.go index 71328df2a..9d705ca05 100644 --- a/pkg/compiler/syscall_test.go +++ b/pkg/compiler/syscall_test.go @@ -4,7 +4,9 @@ import ( "math/big" "testing" + istorage "github.com/nspcc-dev/neo-go/pkg/core/interop/storage" "github.com/nspcc-dev/neo-go/pkg/interop/contract" + "github.com/nspcc-dev/neo-go/pkg/interop/storage" "github.com/nspcc-dev/neo-go/pkg/smartcontract/callflag" "github.com/nspcc-dev/neo-go/pkg/vm/stackitem" "github.com/stretchr/testify/assert" @@ -23,6 +25,13 @@ func TestCallFlags(t *testing.T) { require.EqualValues(t, contract.NoneFlag, callflag.NoneFlag) } +func TestFindFlags(t *testing.T) { + require.EqualValues(t, storage.None, istorage.FindDefault) + require.EqualValues(t, storage.KeysOnly, istorage.FindKeysOnly) + require.EqualValues(t, storage.RemovePrefix, istorage.FindRemovePrefix) + require.EqualValues(t, storage.ValuesOnly, istorage.FindValuesOnly) +} + func TestStoragePutGet(t *testing.T) { src := ` package foo diff --git a/pkg/core/interop/storage/find.go b/pkg/core/interop/storage/find.go new file mode 100644 index 000000000..fa61d89be --- /dev/null +++ b/pkg/core/interop/storage/find.go @@ -0,0 +1,51 @@ +package storage + +import "github.com/nspcc-dev/neo-go/pkg/vm/stackitem" + +// Storage iterator options. +const ( + FindDefault = 0 + FindKeysOnly = 1 << 0 + FindRemovePrefix = 1 << 1 + FindValuesOnly = 1 << 2 + + FindAll = FindDefault | FindKeysOnly | FindRemovePrefix | FindValuesOnly +) + +type Iterator struct { + m []stackitem.MapElement + opts int64 + index int +} + +func NewIterator(m *stackitem.Map, opts int64) *Iterator { + return &Iterator{ + m: m.Value().([]stackitem.MapElement), + opts: opts, + index: -1, + } +} + +func (s *Iterator) Next() bool { + if s.index < len(s.m) { + s.index += 1 + } + return s.index < len(s.m) +} + +func (s *Iterator) Value() stackitem.Item { + key := s.m[s.index].Key.Value().([]byte) + if s.opts&FindRemovePrefix != 0 { + key = key[1:] + } + if s.opts&FindKeysOnly != 0 { + return stackitem.NewByteArray(key) + } + if s.opts&FindValuesOnly != 0 { + return s.m[s.index].Value + } + return stackitem.NewStruct([]stackitem.Item{ + stackitem.NewByteArray(key), + s.m[s.index].Value, + }) +} diff --git a/pkg/core/interop_neo.go b/pkg/core/interop_neo.go index adf7d3f6a..f200296c8 100644 --- a/pkg/core/interop_neo.go +++ b/pkg/core/interop_neo.go @@ -7,11 +7,14 @@ import ( "sort" "github.com/nspcc-dev/neo-go/pkg/core/interop" - "github.com/nspcc-dev/neo-go/pkg/vm" + "github.com/nspcc-dev/neo-go/pkg/core/interop/storage" "github.com/nspcc-dev/neo-go/pkg/vm/stackitem" ) -var errGasLimitExceeded = errors.New("gas limit exceeded") +var ( + errGasLimitExceeded = errors.New("gas limit exceeded") + errFindInvalidOptions = errors.New("invalid Find options") +) // storageFind finds stored key-value pair. func storageFind(ic *interop.Context) error { @@ -21,6 +24,14 @@ func storageFind(ic *interop.Context) error { return fmt.Errorf("%T is not a StorageContext", stcInterface) } prefix := ic.VM.Estack().Pop().Bytes() + opts := ic.VM.Estack().Pop().BigInt().Int64() + if opts&^storage.FindAll != 0 { + return fmt.Errorf("%w: unknown flag", errFindInvalidOptions) + } + if opts&storage.FindValuesOnly != 0 && + opts&(storage.FindKeysOnly|storage.FindRemovePrefix) != 0 { + return fmt.Errorf("%w: KeysOnly conflicts with ValuesOnly", errFindInvalidOptions) + } siMap, err := ic.DAO.GetStorageItemsWithPrefix(stc.ID, prefix) if err != nil { return err @@ -35,8 +46,8 @@ func storageFind(ic *interop.Context) error { filteredMap.Value().([]stackitem.MapElement)[j].Key.Value().([]byte)) == -1 }) - item := vm.NewMapIterator(filteredMap) - ic.VM.Estack().PushVal(item) + item := storage.NewIterator(filteredMap, opts) + ic.VM.Estack().PushVal(stackitem.NewInterop(item)) return nil } diff --git a/pkg/core/interop_neo_test.go b/pkg/core/interop_neo_test.go index 8c2bdf7db..82f11502e 100644 --- a/pkg/core/interop_neo_test.go +++ b/pkg/core/interop_neo_test.go @@ -8,6 +8,7 @@ import ( "github.com/nspcc-dev/neo-go/pkg/core/dao" "github.com/nspcc-dev/neo-go/pkg/core/interop" "github.com/nspcc-dev/neo-go/pkg/core/interop/iterator" + istorage "github.com/nspcc-dev/neo-go/pkg/core/interop/storage" "github.com/nspcc-dev/neo-go/pkg/core/state" "github.com/nspcc-dev/neo-go/pkg/core/storage" "github.com/nspcc-dev/neo-go/pkg/core/transaction" @@ -58,8 +59,9 @@ func TestStorageFind(t *testing.T) { require.NoError(t, err) } - t.Run("normal invocation", func(t *testing.T) { - v.Estack().PushVal([]byte{0x01}) + testFind := func(t *testing.T, prefix byte, opts int64, expected []stackitem.Item) { + v.Estack().PushVal(opts) + v.Estack().PushVal([]byte{prefix}) v.Estack().PushVal(stackitem.NewInterop(&StorageContext{ID: id})) err := storageFind(context) @@ -68,38 +70,71 @@ func TestStorageFind(t *testing.T) { var iter *stackitem.Interop require.NotPanics(t, func() { iter = v.Estack().Pop().Interop() }) - for _, id := range []int{2, 0} { // sorted indices with mathing prefix + for i := range expected { // sorted indices with mathing prefix v.Estack().PushVal(iter) require.NoError(t, iterator.Next(context)) require.True(t, v.Estack().Pop().Bool()) v.Estack().PushVal(iter) require.NoError(t, iterator.Value(context)) - - kv, ok := v.Estack().Pop().Value().([]stackitem.Item) - require.True(t, ok) - require.Len(t, kv, 2) - require.Equal(t, skeys[id], kv[0].Value()) - require.Equal(t, items[id].Value, kv[1].Value()) + require.Equal(t, expected[i], v.Estack().Pop().Item()) } v.Estack().PushVal(iter) require.NoError(t, iterator.Next(context)) require.False(t, v.Estack().Pop().Bool()) + } + + t.Run("normal invocation", func(t *testing.T) { + testFind(t, 0x01, istorage.FindDefault, []stackitem.Item{ + stackitem.NewStruct([]stackitem.Item{ + stackitem.NewByteArray(skeys[2]), + stackitem.NewByteArray(items[2].Value), + }), + stackitem.NewStruct([]stackitem.Item{ + stackitem.NewByteArray(skeys[0]), + stackitem.NewByteArray(items[0].Value), + }), + }) + }) + + t.Run("keys only", func(t *testing.T) { + testFind(t, 0x01, istorage.FindKeysOnly, []stackitem.Item{ + stackitem.NewByteArray(skeys[2]), + stackitem.NewByteArray(skeys[0]), + }) + }) + t.Run("remove prefix", func(t *testing.T) { + testFind(t, 0x01, istorage.FindKeysOnly|istorage.FindRemovePrefix, []stackitem.Item{ + stackitem.NewByteArray(skeys[2][1:]), + stackitem.NewByteArray(skeys[0][1:]), + }) + }) + t.Run("values only", func(t *testing.T) { + testFind(t, 0x01, istorage.FindValuesOnly, []stackitem.Item{ + stackitem.NewByteArray(items[2].Value), + stackitem.NewByteArray(items[0].Value), + }) }) t.Run("normal invocation, empty result", func(t *testing.T) { - v.Estack().PushVal([]byte{0x03}) - v.Estack().PushVal(stackitem.NewInterop(&StorageContext{ID: id})) - - err := storageFind(context) - require.NoError(t, err) - - require.NoError(t, iterator.Next(context)) - require.False(t, v.Estack().Pop().Bool()) + testFind(t, 0x03, istorage.FindDefault, nil) }) + t.Run("invalid options", func(t *testing.T) { + invalid := []int64{ + istorage.FindKeysOnly | istorage.FindValuesOnly, + ^istorage.FindAll, + } + for _, opts := range invalid { + v.Estack().PushVal(opts) + v.Estack().PushVal([]byte{0x01}) + v.Estack().PushVal(stackitem.NewInterop(&StorageContext{ID: id})) + require.Error(t, storageFind(context)) + } + }) t.Run("invalid type for StorageContext", func(t *testing.T) { + v.Estack().PushVal(istorage.FindDefault) v.Estack().PushVal([]byte{0x01}) v.Estack().PushVal(stackitem.NewInterop(nil)) @@ -109,6 +144,7 @@ func TestStorageFind(t *testing.T) { t.Run("invalid id", func(t *testing.T) { invalidID := id + 1 + v.Estack().PushVal(istorage.FindDefault) v.Estack().PushVal([]byte{0x01}) v.Estack().PushVal(stackitem.NewInterop(&StorageContext{ID: invalidID})) diff --git a/pkg/interop/iterator/iterator.go b/pkg/interop/iterator/iterator.go index ff124bb84..396a42dba 100644 --- a/pkg/interop/iterator/iterator.go +++ b/pkg/interop/iterator/iterator.go @@ -28,6 +28,7 @@ func Next(it Iterator) bool { // successful Next call. This function uses `System.Iterator.Value` syscall. // For slices the result is just value. // For maps the result can be casted to a slice of 2 elements: key and value. +// For storage iterators refer to `storage.FindFlags` documentation. func Value(it Iterator) interface{} { return nil } diff --git a/pkg/interop/storage/storage.go b/pkg/interop/storage/storage.go index 8ee4ca0e2..d641e352d 100644 --- a/pkg/interop/storage/storage.go +++ b/pkg/interop/storage/storage.go @@ -14,6 +14,20 @@ import "github.com/nspcc-dev/neo-go/pkg/interop/iterator" // to Neo .net framework's StorageContext class. type Context struct{} +// FindFlags represents parameters to `Find` iterator. +type FindFlags byte + +const ( + // None is default option. Iterator values are key-value pairs. + None FindFlags = 0 + // KeysOnly is used for iterating over keys. + KeysOnly FindFlags = 1 << 0 + // RemovePrefix is used for stripping 1-byte prefix from keys. + RemovePrefix FindFlags = 1 << 1 + // ValuesOnly is used for iterating over values. + ValuesOnly FindFlags = 1 << 2 +) + // ConvertContextToReadOnly returns new context from the given one, but with // writing capability turned off, so that you could only invoke Get and Find // using this new Context. If Context is already read-only this function is a @@ -58,4 +72,6 @@ func Delete(ctx Context, key interface{}) {} // that match the given key (contain it as a prefix). See Put documentation on // possible key types and iterator package documentation on how to use the // returned value. This function uses `System.Storage.Find` syscall. -func Find(ctx Context, key interface{}) iterator.Iterator { return iterator.Iterator{} } +func Find(ctx Context, key interface{}, options FindFlags) iterator.Iterator { + return iterator.Iterator{} +} From 44af99fd079139c2b7fa9f714cbe341d7d0b1d8d Mon Sep 17 00:00:00 2001 From: Evgeniy Stratonikov Date: Tue, 12 Jan 2021 15:09:05 +0300 Subject: [PATCH 4/7] core: add Deserialize flag to `Storage.Find` Allow to deserialize values being iterated over. --- pkg/compiler/syscall_test.go | 1 + pkg/core/interop/storage/find.go | 17 ++++++++++++++--- pkg/core/interop_neo.go | 3 +++ pkg/core/interop_neo_test.go | 31 ++++++++++++++++++++++++++++++- pkg/interop/storage/storage.go | 3 +++ 5 files changed, 51 insertions(+), 4 deletions(-) diff --git a/pkg/compiler/syscall_test.go b/pkg/compiler/syscall_test.go index 9d705ca05..a9564608a 100644 --- a/pkg/compiler/syscall_test.go +++ b/pkg/compiler/syscall_test.go @@ -30,6 +30,7 @@ func TestFindFlags(t *testing.T) { require.EqualValues(t, storage.KeysOnly, istorage.FindKeysOnly) require.EqualValues(t, storage.RemovePrefix, istorage.FindRemovePrefix) require.EqualValues(t, storage.ValuesOnly, istorage.FindValuesOnly) + require.EqualValues(t, storage.DeserializeValues, istorage.FindDeserialize) } func TestStoragePutGet(t *testing.T) { diff --git a/pkg/core/interop/storage/find.go b/pkg/core/interop/storage/find.go index fa61d89be..290b43ff3 100644 --- a/pkg/core/interop/storage/find.go +++ b/pkg/core/interop/storage/find.go @@ -8,8 +8,10 @@ const ( FindKeysOnly = 1 << 0 FindRemovePrefix = 1 << 1 FindValuesOnly = 1 << 2 + FindDeserialize = 1 << 3 - FindAll = FindDefault | FindKeysOnly | FindRemovePrefix | FindValuesOnly + FindAll = FindDefault | FindKeysOnly | FindRemovePrefix | FindValuesOnly | + FindDeserialize ) type Iterator struct { @@ -41,11 +43,20 @@ func (s *Iterator) Value() stackitem.Item { if s.opts&FindKeysOnly != 0 { return stackitem.NewByteArray(key) } + value := s.m[s.index].Value + if s.opts&FindDeserialize != 0 { + bs := s.m[s.index].Value.Value().([]byte) + var err error + value, err = stackitem.DeserializeItem(bs) + if err != nil { + panic(err) + } + } if s.opts&FindValuesOnly != 0 { - return s.m[s.index].Value + return value } return stackitem.NewStruct([]stackitem.Item{ stackitem.NewByteArray(key), - s.m[s.index].Value, + value, }) } diff --git a/pkg/core/interop_neo.go b/pkg/core/interop_neo.go index f200296c8..1a8a09eb7 100644 --- a/pkg/core/interop_neo.go +++ b/pkg/core/interop_neo.go @@ -28,6 +28,9 @@ func storageFind(ic *interop.Context) error { if opts&^storage.FindAll != 0 { return fmt.Errorf("%w: unknown flag", errFindInvalidOptions) } + if opts&storage.FindKeysOnly != 0 && opts&storage.FindDeserialize != 0 { + return fmt.Errorf("%w KeysOnly conflicts with other options", errFindInvalidOptions) + } if opts&storage.FindValuesOnly != 0 && opts&(storage.FindKeysOnly|storage.FindRemovePrefix) != 0 { return fmt.Errorf("%w: KeysOnly conflicts with ValuesOnly", errFindInvalidOptions) diff --git a/pkg/core/interop_neo_test.go b/pkg/core/interop_neo_test.go index 82f11502e..017a23d67 100644 --- a/pkg/core/interop_neo_test.go +++ b/pkg/core/interop_neo_test.go @@ -37,7 +37,8 @@ func TestStorageFind(t *testing.T) { v, contractState, context, chain := createVMAndContractState(t) defer chain.Close() - skeys := [][]byte{{0x01, 0x02}, {0x02, 0x01}, {0x01, 0x01}} + skeys := [][]byte{{0x01, 0x02}, {0x02, 0x01}, {0x01, 0x01}, + {0x04, 0x00}, {0x05, 0x00}} items := []*state.StorageItem{ { Value: []byte{0x01, 0x02, 0x03, 0x04}, @@ -48,6 +49,12 @@ func TestStorageFind(t *testing.T) { { Value: []byte{0x03, 0x04, 0x05, 0x06}, }, + { + Value: []byte{byte(stackitem.ByteArrayT), 2, 0xCA, 0xFE}, + }, + { + Value: []byte{0xFF, 0xFF}, + }, } require.NoError(t, chain.contracts.Management.PutContractState(chain.dao, contractState)) @@ -116,6 +123,27 @@ func TestStorageFind(t *testing.T) { stackitem.NewByteArray(items[0].Value), }) }) + t.Run("deserialize values", func(t *testing.T) { + testFind(t, 0x04, istorage.FindValuesOnly|istorage.FindDeserialize, []stackitem.Item{ + stackitem.NewByteArray(items[3].Value[2:]), + }) + t.Run("invalid", func(t *testing.T) { + v.Estack().PushVal(istorage.FindDeserialize) + v.Estack().PushVal([]byte{0x05}) + v.Estack().PushVal(stackitem.NewInterop(&StorageContext{ID: id})) + err := storageFind(context) + require.NoError(t, err) + + var iter *stackitem.Interop + require.NotPanics(t, func() { iter = v.Estack().Pop().Interop() }) + + v.Estack().PushVal(iter) + require.NoError(t, iterator.Next(context)) + + v.Estack().PushVal(iter) + require.Panics(t, func() { _ = iterator.Value(context) }) + }) + }) t.Run("normal invocation, empty result", func(t *testing.T) { testFind(t, 0x03, istorage.FindDefault, nil) @@ -125,6 +153,7 @@ func TestStorageFind(t *testing.T) { invalid := []int64{ istorage.FindKeysOnly | istorage.FindValuesOnly, ^istorage.FindAll, + istorage.FindKeysOnly | istorage.FindDeserialize, } for _, opts := range invalid { v.Estack().PushVal(opts) diff --git a/pkg/interop/storage/storage.go b/pkg/interop/storage/storage.go index d641e352d..4bf794d54 100644 --- a/pkg/interop/storage/storage.go +++ b/pkg/interop/storage/storage.go @@ -26,6 +26,9 @@ const ( RemovePrefix FindFlags = 1 << 1 // ValuesOnly is used for iterating over values. ValuesOnly FindFlags = 1 << 2 + // DeserializeValues is used for deserializing values on-the-fly. + // It can be combined with other options. + DeserializeValues FindFlags = 1 << 3 ) // ConvertContextToReadOnly returns new context from the given one, but with From 9b1a7021baadbe2827f00ee3c1294fda7f8b6f1d Mon Sep 17 00:00:00 2001 From: Evgeniy Stratonikov Date: Tue, 12 Jan 2021 15:32:27 +0300 Subject: [PATCH 5/7] core: add PickN flags to `Storage.Find` Allow to pick items by index from serialized struct or array. --- pkg/compiler/syscall_test.go | 2 ++ pkg/core/interop/storage/find.go | 9 ++++++- pkg/core/interop_neo.go | 9 ++++++- pkg/core/interop_neo_test.go | 44 +++++++++++++++++++++++++++++++- pkg/interop/storage/storage.go | 4 +++ 5 files changed, 65 insertions(+), 3 deletions(-) diff --git a/pkg/compiler/syscall_test.go b/pkg/compiler/syscall_test.go index a9564608a..9db1af6c3 100644 --- a/pkg/compiler/syscall_test.go +++ b/pkg/compiler/syscall_test.go @@ -31,6 +31,8 @@ func TestFindFlags(t *testing.T) { require.EqualValues(t, storage.RemovePrefix, istorage.FindRemovePrefix) require.EqualValues(t, storage.ValuesOnly, istorage.FindValuesOnly) require.EqualValues(t, storage.DeserializeValues, istorage.FindDeserialize) + require.EqualValues(t, storage.PickField0, istorage.FindPick0) + require.EqualValues(t, storage.PickField1, istorage.FindPick1) } func TestStoragePutGet(t *testing.T) { diff --git a/pkg/core/interop/storage/find.go b/pkg/core/interop/storage/find.go index 290b43ff3..50fd1af92 100644 --- a/pkg/core/interop/storage/find.go +++ b/pkg/core/interop/storage/find.go @@ -9,9 +9,11 @@ const ( FindRemovePrefix = 1 << 1 FindValuesOnly = 1 << 2 FindDeserialize = 1 << 3 + FindPick0 = 1 << 4 + FindPick1 = 1 << 5 FindAll = FindDefault | FindKeysOnly | FindRemovePrefix | FindValuesOnly | - FindDeserialize + FindDeserialize | FindPick0 | FindPick1 ) type Iterator struct { @@ -52,6 +54,11 @@ func (s *Iterator) Value() stackitem.Item { panic(err) } } + if s.opts&FindPick0 != 0 { + value = value.Value().([]stackitem.Item)[0] + } else if s.opts&FindPick1 != 0 { + value = value.Value().([]stackitem.Item)[1] + } if s.opts&FindValuesOnly != 0 { return value } diff --git a/pkg/core/interop_neo.go b/pkg/core/interop_neo.go index 1a8a09eb7..98671e6ef 100644 --- a/pkg/core/interop_neo.go +++ b/pkg/core/interop_neo.go @@ -28,13 +28,20 @@ func storageFind(ic *interop.Context) error { if opts&^storage.FindAll != 0 { return fmt.Errorf("%w: unknown flag", errFindInvalidOptions) } - if opts&storage.FindKeysOnly != 0 && opts&storage.FindDeserialize != 0 { + if opts&storage.FindKeysOnly != 0 && + opts&(storage.FindDeserialize|storage.FindPick0|storage.FindPick1) != 0 { return fmt.Errorf("%w KeysOnly conflicts with other options", errFindInvalidOptions) } if opts&storage.FindValuesOnly != 0 && opts&(storage.FindKeysOnly|storage.FindRemovePrefix) != 0 { return fmt.Errorf("%w: KeysOnly conflicts with ValuesOnly", errFindInvalidOptions) } + if opts&storage.FindPick0 != 0 && opts&storage.FindPick1 != 0 { + return fmt.Errorf("%w: Pick0 conflicts with Pick1", errFindInvalidOptions) + } + if opts&storage.FindDeserialize == 0 && (opts&storage.FindPick0 != 0 || opts&storage.FindPick1 != 0) { + return fmt.Errorf("%w: PickN is specified without Deserialize", errFindInvalidOptions) + } siMap, err := ic.DAO.GetStorageItemsWithPrefix(stc.ID, prefix) if err != nil { return err diff --git a/pkg/core/interop_neo_test.go b/pkg/core/interop_neo_test.go index 017a23d67..a7763ee24 100644 --- a/pkg/core/interop_neo_test.go +++ b/pkg/core/interop_neo_test.go @@ -1,6 +1,7 @@ package core import ( + "math/big" "testing" "github.com/nspcc-dev/neo-go/pkg/config/netmode" @@ -37,8 +38,20 @@ func TestStorageFind(t *testing.T) { v, contractState, context, chain := createVMAndContractState(t) defer chain.Close() + arr := []stackitem.Item{ + stackitem.NewBigInteger(big.NewInt(42)), + stackitem.NewByteArray([]byte("second")), + stackitem.Null{}, + } + rawArr, err := stackitem.SerializeItem(stackitem.NewArray(arr)) + require.NoError(t, err) + rawArr0, err := stackitem.SerializeItem(stackitem.NewArray(arr[:0])) + require.NoError(t, err) + rawArr1, err := stackitem.SerializeItem(stackitem.NewArray(arr[:1])) + require.NoError(t, err) + skeys := [][]byte{{0x01, 0x02}, {0x02, 0x01}, {0x01, 0x01}, - {0x04, 0x00}, {0x05, 0x00}} + {0x04, 0x00}, {0x05, 0x00}, {0x06}, {0x07}, {0x08}} items := []*state.StorageItem{ { Value: []byte{0x01, 0x02, 0x03, 0x04}, @@ -55,6 +68,15 @@ func TestStorageFind(t *testing.T) { { Value: []byte{0xFF, 0xFF}, }, + { + Value: rawArr, + }, + { + Value: rawArr0, + }, + { + Value: rawArr1, + }, } require.NoError(t, chain.contracts.Management.PutContractState(chain.dao, contractState)) @@ -83,6 +105,10 @@ func TestStorageFind(t *testing.T) { require.True(t, v.Estack().Pop().Bool()) v.Estack().PushVal(iter) + if expected[i] == nil { + require.Panics(t, func() { _ = iterator.Value(context) }) + return + } require.NoError(t, iterator.Value(context)) require.Equal(t, expected[i], v.Estack().Pop().Item()) } @@ -144,6 +170,19 @@ func TestStorageFind(t *testing.T) { require.Panics(t, func() { _ = iterator.Value(context) }) }) }) + t.Run("PickN", func(t *testing.T) { + testFind(t, 0x06, istorage.FindPick0|istorage.FindValuesOnly|istorage.FindDeserialize, arr[:1]) + testFind(t, 0x06, istorage.FindPick1|istorage.FindValuesOnly|istorage.FindDeserialize, arr[1:2]) + // Array with 0 elements. + testFind(t, 0x07, istorage.FindPick0|istorage.FindValuesOnly|istorage.FindDeserialize, + []stackitem.Item{nil}) + // Array with 1 element. + testFind(t, 0x08, istorage.FindPick1|istorage.FindValuesOnly|istorage.FindDeserialize, + []stackitem.Item{nil}) + // Not an array, but serialized ByteArray. + testFind(t, 0x04, istorage.FindPick1|istorage.FindValuesOnly|istorage.FindDeserialize, + []stackitem.Item{nil}) + }) t.Run("normal invocation, empty result", func(t *testing.T) { testFind(t, 0x03, istorage.FindDefault, nil) @@ -154,6 +193,9 @@ func TestStorageFind(t *testing.T) { istorage.FindKeysOnly | istorage.FindValuesOnly, ^istorage.FindAll, istorage.FindKeysOnly | istorage.FindDeserialize, + istorage.FindPick0, + istorage.FindPick0 | istorage.FindPick1 | istorage.FindDeserialize, + istorage.FindPick0 | istorage.FindPick1, } for _, opts := range invalid { v.Estack().PushVal(opts) diff --git a/pkg/interop/storage/storage.go b/pkg/interop/storage/storage.go index 4bf794d54..521c9888a 100644 --- a/pkg/interop/storage/storage.go +++ b/pkg/interop/storage/storage.go @@ -29,6 +29,10 @@ const ( // DeserializeValues is used for deserializing values on-the-fly. // It can be combined with other options. DeserializeValues FindFlags = 1 << 3 + // PickField0 is used to get first field in a serialized struct or array. + PickField0 FindFlags = 1 << 4 + // PickField1 is used to get second field in a serialized struct or array. + PickField1 FindFlags = 1 << 5 ) // ConvertContextToReadOnly returns new context from the given one, but with From 50d515a3e5012398c1771ddaeff351bacbbf7200 Mon Sep 17 00:00:00 2001 From: Evgeniy Stratonikov Date: Tue, 12 Jan 2021 15:52:08 +0300 Subject: [PATCH 6/7] cli: add tests for Storage.Find invocations It returns values of different types we better be sure that compiler emits correct code. --- cli/contract_test.go | 38 +++++++++++++++++++++++++++++++++++++ cli/testdata/deploy/main.go | 15 +++++++++++++++ 2 files changed, 53 insertions(+) diff --git a/cli/contract_test.go b/cli/contract_test.go index 4db8aca80..f8886d33c 100644 --- a/cli/contract_test.go +++ b/cli/contract_test.go @@ -6,11 +6,13 @@ import ( "io/ioutil" "os" "path" + "strconv" "strings" "testing" "github.com/nspcc-dev/neo-go/internal/random" "github.com/nspcc-dev/neo-go/pkg/config" + "github.com/nspcc-dev/neo-go/pkg/core/interop/storage" "github.com/nspcc-dev/neo-go/pkg/core/state" "github.com/nspcc-dev/neo-go/pkg/encoding/address" "github.com/nspcc-dev/neo-go/pkg/encoding/fixedn" @@ -18,6 +20,7 @@ import ( "github.com/nspcc-dev/neo-go/pkg/smartcontract/nef" "github.com/nspcc-dev/neo-go/pkg/util" "github.com/nspcc-dev/neo-go/pkg/vm" + "github.com/nspcc-dev/neo-go/pkg/vm/stackitem" "github.com/stretchr/testify/require" ) @@ -254,6 +257,41 @@ func TestComlileAndInvokeFunction(t *testing.T) { }) }) + t.Run("test Storage.Find", func(t *testing.T) { + cmd := []string{"neo-go", "contract", "testinvokefunction", + "--rpc-endpoint", "http://" + e.RPC.Addr, + h.StringLE(), "testFind"} + + t.Run("keys only", func(t *testing.T) { + e.Run(t, append(cmd, strconv.FormatInt(storage.FindKeysOnly, 10))...) + res := new(result.Invoke) + require.NoError(t, json.Unmarshal(e.Out.Bytes(), res)) + require.Equal(t, vm.HaltState.String(), res.State) + require.Len(t, res.Stack, 1) + require.Equal(t, []stackitem.Item{ + stackitem.Make("findkey1"), + stackitem.Make("findkey2"), + }, res.Stack[0].Value()) + }) + t.Run("both", func(t *testing.T) { + e.Run(t, append(cmd, strconv.FormatInt(storage.FindDefault, 10))...) + res := new(result.Invoke) + require.NoError(t, json.Unmarshal(e.Out.Bytes(), res)) + require.Equal(t, vm.HaltState.String(), res.State) + require.Len(t, res.Stack, 1) + + arr, ok := res.Stack[0].Value().([]stackitem.Item) + require.True(t, ok) + require.Len(t, arr, 2) + require.Equal(t, []stackitem.Item{ + stackitem.Make("findkey1"), stackitem.Make("value1"), + }, arr[0].Value()) + require.Equal(t, []stackitem.Item{ + stackitem.Make("findkey2"), stackitem.Make("value2"), + }, arr[1].Value()) + }) + }) + t.Run("Update", func(t *testing.T) { nefName := path.Join(tmpDir, "updated.nef") manifestName := path.Join(tmpDir, "updated.manifest.json") diff --git a/cli/testdata/deploy/main.go b/cli/testdata/deploy/main.go index dcadc30ea..514c3e448 100644 --- a/cli/testdata/deploy/main.go +++ b/cli/testdata/deploy/main.go @@ -4,6 +4,7 @@ import ( "github.com/nspcc-dev/neo-go/cli/testdata/deploy/sub" "github.com/nspcc-dev/neo-go/pkg/interop" "github.com/nspcc-dev/neo-go/pkg/interop/contract" + "github.com/nspcc-dev/neo-go/pkg/interop/iterator" "github.com/nspcc-dev/neo-go/pkg/interop/runtime" "github.com/nspcc-dev/neo-go/pkg/interop/storage" ) @@ -46,3 +47,17 @@ func GetValue() string { val2 := storage.Get(ctx, sub.Key) return val1.(string) + "|" + val2.(string) } + +// TestFind finds items with the specified prefix. +func TestFind(f storage.FindFlags) []interface{} { + ctx := storage.GetContext() + storage.Put(ctx, "findkey1", "value1") + storage.Put(ctx, "findkey2", "value2") + + var result []interface{} + iter := storage.Find(ctx, "findkey", f) + for iterator.Next(iter) { + result = append(result, iterator.Value(iter)) + } + return result +} From db122de1975d35d6637ba19f24fd91e08a75eb68 Mon Sep 17 00:00:00 2001 From: Roman Khimov Date: Fri, 15 Jan 2021 21:06:35 +0300 Subject: [PATCH 7/7] storage: fix linter warnings pkg/core/interop/storage/find.go:19:6: exported type Iterator should have comment or be unexported pkg/core/interop/storage/find.go:25:1: exported function NewIterator should have comment or be unexported pkg/core/interop/storage/find.go:33:1: exported method Iterator.Next should have comment or be unexported pkg/core/interop/storage/find.go:35:3: should replace s.index += 1 with s.index++ pkg/core/interop/storage/find.go:40:1: exported method Iterator.Value should have comment or be unexported --- pkg/core/interop/storage/find.go | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/pkg/core/interop/storage/find.go b/pkg/core/interop/storage/find.go index 50fd1af92..a3b30cea9 100644 --- a/pkg/core/interop/storage/find.go +++ b/pkg/core/interop/storage/find.go @@ -16,12 +16,14 @@ const ( FindDeserialize | FindPick0 | FindPick1 ) +// Iterator is an iterator state representation. type Iterator struct { m []stackitem.MapElement opts int64 index int } +// NewIterator creates a new Iterator with given options for a given map. func NewIterator(m *stackitem.Map, opts int64) *Iterator { return &Iterator{ m: m.Value().([]stackitem.MapElement), @@ -30,13 +32,17 @@ func NewIterator(m *stackitem.Map, opts int64) *Iterator { } } +// Next advances the iterator and returns true if Value can be called at the +// current position. func (s *Iterator) Next() bool { if s.index < len(s.m) { - s.index += 1 + s.index++ } return s.index < len(s.m) } +// Value returns current iterators value (exact type depends on options this +// iterator was created with). func (s *Iterator) Value() stackitem.Item { key := s.m[s.index].Key.Value().([]byte) if s.opts&FindRemovePrefix != 0 {