stackitem: return error in TryBytes() for big byte strings

Follow neo-project/neo-vm#349.

pkg/core/blockchain_test.go

@@ -354,6 +354,17 @@ func TestVerifyHashAgainstScript(t *testing.T) {
 		err := bc.verifyHashAgainstScript(hash.Hash160(verif), w, ic, false, gas)
 		require.True(t, errors.Is(err, ErrVerificationFailed))
 	})
+	t.Run("BadResult", func(t *testing.T) {
+		verif := make([]byte, 66)
+		verif[0] = byte(opcode.PUSHDATA1)
+		verif[1] = 64
+		w := &transaction.Witness{
+			InvocationScript:   []byte{byte(opcode.NOP)},
+			VerificationScript: verif,
+		}
+		err := bc.verifyHashAgainstScript(hash.Hash160(verif), w, ic, false, gas)
+		require.True(t, errors.Is(err, ErrVerificationFailed))
+	})
 	t.Run("TooManyResults", func(t *testing.T) {
 		verif := []byte{byte(opcode.NOP)}
 		w := &transaction.Witness{

pkg/vm/stackitem/item.go

@@ -507,7 +507,7 @@ func (i *ByteArray) String() string {
 // TryBool implements Item interface.
 func (i *ByteArray) TryBool() (bool, error) {
 	if len(i.value) > MaxBigIntegerSizeBits/8 {
-		return true, nil
+		return false, errors.New("too big byte string")
 	}
 	for _, b := range i.value {
 		if b != 0 {

pkg/vm/testdata/neo-vm

@@ -1 +1 @@
-Subproject commit 359e8631ee2ddfefe8261afcec1a5bab9d9bddf9
+Subproject commit 377464ed475a3de108e1bf9c834bd2279b72624e