vm: handle very big int creation properly

Determine size as in reference implementation instead of
`big.Int.BitLen()`

Close #1437.

pkg/vm/stackitem/item.go

@@ -350,8 +350,19 @@ type BigInteger struct {
 
 // NewBigInteger returns an new BigInteger object.
 func NewBigInteger(value *big.Int) *BigInteger {
-	if value.BitLen() > MaxBigIntegerSizeBits {
+	const tooBigErrMsg = "integer is too big"
-		panic("integer is too big")
+
+	// There are 2 cases, when `BitLen` differs from actual size:
+	// 1. Positive integer with highest bit on byte boundary = 1.
+	// 2. Negative integer with highest bit on byte boundary = 1
+	//    minus some value. (-0x80 -> 0x80, -0x7F -> 0x81, -0x81 -> 0x7FFF).
+	sz := value.BitLen()
+	if sz > MaxBigIntegerSizeBits {
+		panic(tooBigErrMsg)
+	} else if sz == MaxBigIntegerSizeBits {
+		if value.Sign() == 1 || value.TrailingZeroBits() != MaxBigIntegerSizeBits-1 {
+			panic(tooBigErrMsg)
+		}
 	}
 	return &BigInteger{
 		value: value,

pkg/vm/stackitem/item_test.go

@@ -4,6 +4,7 @@ import (
 	"math/big"
 	"testing"
 
+	"github.com/nspcc-dev/neo-go/pkg/encoding/bigint"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
@@ -436,6 +437,34 @@ func TestMarshalJSON(t *testing.T) {
 	}
 }
 
+func TestNewVeryBigInteger(t *testing.T) {
+	check := func(ok bool, v *big.Int) {
+		bs := bigint.ToBytes(v)
+		if ok {
+			assert.True(t, len(bs)*8 <= MaxBigIntegerSizeBits)
+		} else {
+			assert.True(t, len(bs)*8 > MaxBigIntegerSizeBits)
+			assert.Panics(t, func() { NewBigInteger(v) })
+		}
+	}
+
+	maxBitSet := big.NewInt(1)
+	maxBitSet.Lsh(maxBitSet, MaxBigIntegerSizeBits-1)
+
+	check(false, maxBitSet)
+	check(true, new(big.Int).Neg(maxBitSet))
+
+	minus1 := new(big.Int).Sub(maxBitSet, big.NewInt(1))
+	check(true, minus1)
+	check(true, new(big.Int).Neg(minus1))
+
+	plus1 := new(big.Int).Add(maxBitSet, big.NewInt(1))
+	check(false, plus1)
+	check(false, new(big.Int).Neg(plus1))
+
+	check(false, new(big.Int).Mul(maxBitSet, big.NewInt(2)))
+}
+
 func TestDeepCopy(t *testing.T) {
 	testCases := []struct {
 		name string

pkg/vm/vm_test.go

@@ -994,7 +994,7 @@ func TestArith(t *testing.T) {
 
 func TestADDBigResult(t *testing.T) {
 	prog := makeProgram(opcode.ADD)
-	runWithArgs(t, prog, nil, getBigInt(stackitem.MaxBigIntegerSizeBits, -1), 1)
+	runWithArgs(t, prog, nil, getBigInt(stackitem.MaxBigIntegerSizeBits-1, -1), 1) // 0x7FFF...
 }
 
 func TestMULBigResult(t *testing.T) {
@@ -1035,7 +1035,9 @@ func TestArithNegativeArguments(t *testing.T) {
 
 func TestSUBBigResult(t *testing.T) {
 	prog := makeProgram(opcode.SUB)
-	runWithArgs(t, prog, nil, getBigInt(stackitem.MaxBigIntegerSizeBits, -1), -1)
+	bi := getBigInt(stackitem.MaxBigIntegerSizeBits-1, -1)
+	runWithArgs(t, prog, new(big.Int).Sub(big.NewInt(-1), bi), -1, bi)
+	runWithArgs(t, prog, nil, -2, bi)
 }
 
 func TestSHR(t *testing.T) {
@@ -1198,7 +1200,7 @@ func TestINC(t *testing.T) {
 func TestINCBigResult(t *testing.T) {
 	prog := makeProgram(opcode.INC, opcode.INC)
 	vm := load(prog)
-	x := getBigInt(stackitem.MaxBigIntegerSizeBits, -2)
+	x := getBigInt(stackitem.MaxBigIntegerSizeBits-1, -2)
 	vm.estack.PushVal(x)
 
 	require.NoError(t, vm.Step())
@@ -1212,7 +1214,7 @@ func TestINCBigResult(t *testing.T) {
 func TestDECBigResult(t *testing.T) {
 	prog := makeProgram(opcode.DEC, opcode.DEC)
 	vm := load(prog)
-	x := getBigInt(stackitem.MaxBigIntegerSizeBits, -2)
+	x := getBigInt(stackitem.MaxBigIntegerSizeBits-1, -1)
 	x.Neg(x)
 	vm.estack.PushVal(x)