vm: limit maximum nesting of exception contexts

Follow neo-project/neo#365. neo-vm submodule is updated just to show the
relevant latest commit, nothing really changed there.

pkg/vm/testdata/neo-vm

@@ -1 +1 @@
-Subproject commit 8476d0abba10b2efdc94e8d4dc27f5c30c8b66e1
+Subproject commit e3f1584b1953dcc13075a57803240858c8a480de

pkg/vm/vm.go

@@ -45,6 +45,10 @@ const (
 	// MaxInvocationStackSize is the maximum size of an invocation stack.
 	MaxInvocationStackSize = 1024
 
+	// MaxTryNestingDepth is the maximum level of TRY nesting allowed,
+	// that is you can't have more exception handling contexts than this.
+	MaxTryNestingDepth = 16
+
 	// MaxStackSize is the maximum number of items allowed to be
 	// on all stacks at once.
 	MaxStackSize = 2 * 1024
@@ -1361,6 +1365,9 @@ func (v *VM) execute(ctx *Context, op opcode.Opcode, parameter []byte) (err erro
 
 	case opcode.TRY, opcode.TRYL:
 		catchP, finallyP := getTryParams(op, parameter)
+		if ctx.tryStack.Len() >= MaxTryNestingDepth {
+			panic("maximum TRY depth exceeded")
+		}
 		cOffset := v.getJumpOffset(ctx, catchP)
 		fOffset := v.getJumpOffset(ctx, finallyP)
 		if cOffset == 0 && fOffset == 0 {

pkg/vm/vm_test.go

@@ -1321,6 +1321,16 @@ func TestTRY(t *testing.T) {
 			inner := getTRYProgram(throw, add5, []byte{byte(opcode.THROW)})
 			getTRYTestFunc(32, inner, add5, add9)(t)
 		})
+		t.Run("TryMaxDepth", func(t *testing.T) {
+			loopTries := []byte{byte(opcode.INITSLOT), 0x01, 0x00,
+				byte(opcode.PUSH16), byte(opcode.INC), byte(opcode.STLOC0),
+				byte(opcode.TRY), 1, 1, // jump target
+				byte(opcode.LDLOC0), byte(opcode.DEC), byte(opcode.DUP),
+				byte(opcode.STLOC0), byte(opcode.PUSH0),
+				byte(opcode.JMPGT), 0xf8, byte(opcode.LDLOC0)}
+			vm := load(loopTries)
+			checkVMFailed(t, vm)
+		})
 	})
 }