rpc: add paging to get*transfer calls

pkg/rpc/server/server.go

@@ -450,24 +450,34 @@ func (s *Server) getVersion(_ request.Params) (interface{}, *response.Error) {
 	}, nil
 }
 
-func getTimestampsAndLimit(ps request.Params, index int) (uint32, uint32, int, error) {
+func getTimestampsAndLimit(ps request.Params, index int) (uint32, uint32, int, int, error) {
 	var start, end uint32
-	var limit int
+	var limit, page int
-	pStart, pEnd, pLimit := ps.Value(index), ps.Value(index+1), ps.Value(index+2)
+	pStart, pEnd, pLimit, pPage := ps.Value(index), ps.Value(index+1), ps.Value(index+2), ps.Value(index+3)
+	if pPage != nil {
+		p, err := pPage.GetInt()
+		if err != nil {
+			return 0, 0, 0, 0, err
+		}
+		if p < 0 {
+			return 0, 0, 0, 0, errors.New("can't use negative page")
+		}
+		page = p
+	}
 	if pLimit != nil {
 		l, err := pLimit.GetInt()
 		if err != nil {
-			return 0, 0, 0, err
+			return 0, 0, 0, 0, err
 		}
 		if l <= 0 {
-			return 0, 0, 0, errors.New("can't use negative or zero limit")
+			return 0, 0, 0, 0, errors.New("can't use negative or zero limit")
 		}
 		limit = l
 	}
 	if pEnd != nil {
 		val, err := pEnd.GetInt()
 		if err != nil {
-			return 0, 0, 0, err
+			return 0, 0, 0, 0, err
 		}
 		end = uint32(val)
 	} else {
@@ -476,13 +486,13 @@ func getTimestampsAndLimit(ps request.Params, index int) (uint32, uint32, int, e
 	if pStart != nil {
 		val, err := pStart.GetInt()
 		if err != nil {
-			return 0, 0, 0, err
+			return 0, 0, 0, 0, err
 		}
 		start = uint32(val)
 	} else {
 		start = uint32(time.Now().Add(-time.Hour * 24 * 7).Unix())
 	}
-	return start, end, limit, nil
+	return start, end, limit, page, nil
 }
 
 func getAssetMaps(name string) (map[util.Uint256]*result.AssetUTXO, map[util.Uint256]*result.AssetUTXO, error) {
@@ -533,7 +543,7 @@ func (s *Server) getUTXOTransfers(ps request.Params) (interface{}, *response.Err
 		index++
 	}
 
-	start, end, limit, err := getTimestampsAndLimit(ps, index)
+	start, end, limit, page, err := getTimestampsAndLimit(ps, index)
 	if err != nil {
 		return nil, response.NewInvalidParamsError("", err)
 	}
@@ -543,21 +553,23 @@ func (s *Server) getUTXOTransfers(ps request.Params) (interface{}, *response.Err
 		return nil, response.NewInvalidParamsError("", err)
 	}
 	tr := new(state.Transfer)
+	var resCount, frameCount int
 	err = s.chain.ForEachTransfer(addr, tr, func() (bool, error) {
+		// Iterating from newest to oldest, not yet reached required
+		// time frame, continue looping.
 		if tr.Timestamp > end {
 			return true, nil
 		}
-		var count int
+		// Iterating from newest to oldest, moved past required
-		for _, res := range sent {
+		// time frame, stop looping.
-			count += len(res.Transactions)
+		if tr.Timestamp < start {
-		}
-		for _, res := range recv {
-			count += len(res.Transactions)
-		}
-		if tr.Timestamp < start ||
-			(limit != 0 && count >= limit) {
 			return false, nil
 		}
+		frameCount++
+		// Using limits, not yet reached required page.
+		if limit != 0 && page*limit >= frameCount {
+			return true, nil
+		}
 		assetID := core.GoverningTokenID()
 		if !tr.IsGoverning {
 			assetID = core.UtilityTokenID()
@@ -576,6 +588,11 @@ func (s *Server) getUTXOTransfers(ps request.Params) (interface{}, *response.Err
 			})
 			a.TotalAmount += tr.Amount
 		}
+		resCount++
+		// Using limits, reached limit.
+		if limit != 0 && resCount >= limit {
+			return false, nil
+		}
 		return true, nil
 	})
 	if err != nil {
@@ -741,7 +758,7 @@ func (s *Server) getNEP5Transfers(ps request.Params) (interface{}, *response.Err
 		return nil, response.ErrInvalidParams
 	}
 
-	start, end, limit, err := getTimestampsAndLimit(ps, 1)
+	start, end, limit, page, err := getTimestampsAndLimit(ps, 1)
 	if err != nil {
 		return nil, response.NewInvalidParamsError("", err)
 	}
@@ -752,14 +769,23 @@ func (s *Server) getNEP5Transfers(ps request.Params) (interface{}, *response.Err
 		Sent:     []result.NEP5Transfer{},
 	}
 	tr := new(state.NEP5Transfer)
+	var resCount, frameCount int
 	err = s.chain.ForEachNEP5Transfer(u, tr, func() (bool, error) {
+		// Iterating from newest to oldest, not yet reached required
+		// time frame, continue looping.
 		if tr.Timestamp > end {
 			return true, nil
 		}
-		if tr.Timestamp < start ||
+		// Iterating from newest to oldest, moved past required
-			(limit != 0 && (len(bs.Received)+len(bs.Sent) >= limit)) {
+		// time frame, stop looping.
+		if tr.Timestamp < start {
 			return false, nil
 		}
+		frameCount++
+		// Using limits, not yet reached required page.
+		if limit != 0 && page*limit >= frameCount {
+			return true, nil
+		}
 		transfer := result.NEP5Transfer{
 			Timestamp: tr.Timestamp,
 			Asset:     tr.Asset,
@@ -774,14 +800,18 @@ func (s *Server) getNEP5Transfers(ps request.Params) (interface{}, *response.Err
 				transfer.Address = address.Uint160ToString(tr.From)
 			}
 			bs.Received = append(bs.Received, transfer)
-			return true, nil
+		} else {
-		}
-
 			transfer.Amount = strconv.FormatInt(-tr.Amount, 10)
 			if !tr.To.Equals(util.Uint160{}) {
 				transfer.Address = address.Uint160ToString(tr.To)
 			}
 			bs.Sent = append(bs.Sent, transfer)
+		}
+		resCount++
+		// Using limits, reached limit.
+		if limit != 0 && resCount >= limit {
+			return false, nil
+		}
 		return true, nil
 	})
 	if err != nil {

pkg/rpc/server/server_test.go

@@ -1164,12 +1164,11 @@ func testRPCProtocol(t *testing.T, doRPCCall func(string, string, *testing.T) []
 	})
 
 	t.Run("getutxotransfers", func(t *testing.T) {
-		testGetUTXO := func(t *testing.T, asset string, start, stop int) {
+		testGetUTXO := func(t *testing.T, asset string, start, stop, limit, page int, present []int) {
 			ps := []string{`"AKkkumHbBipZ46UMZJoFynJMXzSRnBvKcs"`}
 			if asset != "" {
 				ps = append(ps, fmt.Sprintf("%q", asset))
 			}
-			if start >= 0 {
 			if start > int(e.chain.HeaderHeight()) {
 				ps = append(ps, strconv.Itoa(int(time.Now().Unix())))
 			} else {
@@ -1182,6 +1181,11 @@ func testRPCProtocol(t *testing.T, doRPCCall func(string, string, *testing.T) []
 				require.NoError(t, err)
 				ps = append(ps, strconv.Itoa(int(b.Timestamp)))
 			}
+			if limit != 0 {
+				ps = append(ps, strconv.Itoa(limit))
+			}
+			if page != 0 {
+				ps = append(ps, strconv.Itoa(page))
 			}
 			p := strings.Join(ps, ", ")
 			rpc := fmt.Sprintf(`{"jsonrpc": "2.0", "id": 1, "method": "getutxotransfers", "params": [%s]}`, p)
@@ -1189,11 +1193,17 @@ func testRPCProtocol(t *testing.T, doRPCCall func(string, string, *testing.T) []
 			res := checkErrGetResult(t, body, false)
 			actual := new(result.GetUTXO)
 			require.NoError(t, json.Unmarshal(res, actual))
-			checkTransfers(t, e, actual, asset, start, stop)
+			checkTransfers(t, e, actual, present)
 		}
-		t.Run("RestrictByAsset", func(t *testing.T) { testGetUTXO(t, "neo", 0, 0) })
+		// See `checkTransfers` for the last parameter values.
-		t.Run("TooBigStart", func(t *testing.T) { testGetUTXO(t, "", 300, 0) })
+		t.Run("All", func(t *testing.T) { testGetUTXO(t, "", 0, 207, 0, 0, []int{0, 1, 2, 3, 4, 5, 6, 7}) })
-		t.Run("RestrictAll", func(t *testing.T) { testGetUTXO(t, "", 202, 203) })
+		t.Run("RestrictByAsset", func(t *testing.T) { testGetUTXO(t, "neo", 0, 0, 0, 0, []int{0, 1, 2, 6, 7}) })
+		t.Run("TooBigStart", func(t *testing.T) { testGetUTXO(t, "", 300, 0, 0, 0, []int{}) })
+		t.Run("RestrictAll", func(t *testing.T) { testGetUTXO(t, "", 202, 203, 0, 0, []int{1, 2, 3}) })
+		t.Run("Limit", func(t *testing.T) { testGetUTXO(t, "neo", 0, 207, 2, 0, []int{7, 6}) })
+		t.Run("Limit 2", func(t *testing.T) { testGetUTXO(t, "", 0, 204, 1, 0, []int{5}) })
+		t.Run("Limit with page", func(t *testing.T) { testGetUTXO(t, "", 0, 204, 1, 1, []int{4}) })
+		t.Run("Limit with page 2", func(t *testing.T) { testGetUTXO(t, "", 0, 204, 2, 2, []int{1, 0}) })
 	})
 
 	t.Run("getnep5transfers", func(t *testing.T) {
@@ -1312,43 +1322,43 @@ func checkNep5TransfersAux(t *testing.T, e *executor, acc interface{}, onlyFirst
 	require.Equal(t, uint32(0), res.Sent[0].NotifyIndex)
 }
 
-func checkTransfers(t *testing.T, e *executor, acc interface{}, asset string, start, stop int) {
+func checkTransfers(t *testing.T, e *executor, acc interface{}, checked []int) {
+	type transfer struct {
+		sent   bool
+		asset  string
+		index  uint32
+		amount int64
+	}
+
+	var transfers = []transfer{
+		{false, "neo", 1, 99999000},       // NEO to us.
+		{false, "neo", 202, 99999000},     // NEO roundtrip for GAS claim.
+		{true, "neo", 202, 99999000},      // NEO roundtrip for GAS claim.
+		{false, "gas", 203, 160798392000}, // GAS claim.
+		{false, "gas", 204, 150798392000}, // Remainder from contract deployment.
+		{true, "gas", 204, 160798392000},  // Contract deployment.
+		{false, "neo", 206, 99998000},     // Remainder of NEO sent.
+		{true, "neo", 206, 99999000},      // NEO to another validator.
+	}
 	res := acc.(*result.GetUTXO)
 	require.Equal(t, res.Address, "AKkkumHbBipZ46UMZJoFynJMXzSRnBvKcs")
 
-	// transfer from multisig address to us
+	for i, tr := range transfers {
-	u := getUTXOForBlock(res, false, "neo", 1)
+		var present bool
-	if start <= 1 && (stop == 0 || stop >= 1) && (asset == "neo" || asset == "") {
+
+		u := getUTXOForBlock(res, tr.sent, tr.asset, tr.index)
+		for j := range checked {
+			if checked[j] == i {
+				present = true
+				break
+			}
+		}
+		if present {
 			require.NotNil(t, u)
-		require.EqualValues(t, int64(99999000), u.Amount)
+			require.EqualValues(t, tr.amount, u.Amount)
 		} else {
 			require.Nil(t, u)
 		}
-
-	// gas claim
-	u = getUTXOForBlock(res, false, "gas", 203)
-	if start <= 203 && (stop == 0 || stop >= 203) && (asset == "gas" || asset == "") {
-		require.NotNil(t, u)
-		require.EqualValues(t, int64(160798392000), u.Amount)
-	} else {
-		require.Nil(t, u)
-	}
-
-	// transfer from us to another validator
-	u = getUTXOForBlock(res, true, "neo", 206)
-	if start <= 206 && (stop == 0 || stop >= 206) && (asset == "neo" || asset == "") {
-		require.NotNil(t, u)
-		require.EqualValues(t, int64(99999000), u.Amount)
-	} else {
-		require.Nil(t, u)
-	}
-
-	u = getUTXOForBlock(res, false, "neo", 206)
-	if start <= 206 && (stop == 0 || stop >= 206) && (asset == "neo" || asset == "") {
-		require.NotNil(t, u)
-		require.EqualValues(t, int64(99998000), u.Amount)
-	} else {
-		require.Nil(t, u)
 	}
 }