From 56d57611ca6bb09f48354971c0685ed4c9195b6d Mon Sep 17 00:00:00 2001 From: Roman Khimov Date: Sun, 13 Sep 2020 00:12:45 +0300 Subject: [PATCH] rpc: add paging to get*transfer calls --- pkg/rpc/server/server.go | 84 +++++++++++++++++-------- pkg/rpc/server/server_test.go | 112 ++++++++++++++++++---------------- 2 files changed, 118 insertions(+), 78 deletions(-) diff --git a/pkg/rpc/server/server.go b/pkg/rpc/server/server.go index d670ce419..015d162b4 100644 --- a/pkg/rpc/server/server.go +++ b/pkg/rpc/server/server.go @@ -450,24 +450,34 @@ func (s *Server) getVersion(_ request.Params) (interface{}, *response.Error) { }, nil } -func getTimestampsAndLimit(ps request.Params, index int) (uint32, uint32, int, error) { +func getTimestampsAndLimit(ps request.Params, index int) (uint32, uint32, int, int, error) { var start, end uint32 - var limit int - pStart, pEnd, pLimit := ps.Value(index), ps.Value(index+1), ps.Value(index+2) + var limit, page int + pStart, pEnd, pLimit, pPage := ps.Value(index), ps.Value(index+1), ps.Value(index+2), ps.Value(index+3) + if pPage != nil { + p, err := pPage.GetInt() + if err != nil { + return 0, 0, 0, 0, err + } + if p < 0 { + return 0, 0, 0, 0, errors.New("can't use negative page") + } + page = p + } if pLimit != nil { l, err := pLimit.GetInt() if err != nil { - return 0, 0, 0, err + return 0, 0, 0, 0, err } if l <= 0 { - return 0, 0, 0, errors.New("can't use negative or zero limit") + return 0, 0, 0, 0, errors.New("can't use negative or zero limit") } limit = l } if pEnd != nil { val, err := pEnd.GetInt() if err != nil { - return 0, 0, 0, err + return 0, 0, 0, 0, err } end = uint32(val) } else { @@ -476,13 +486,13 @@ func getTimestampsAndLimit(ps request.Params, index int) (uint32, uint32, int, e if pStart != nil { val, err := pStart.GetInt() if err != nil { - return 0, 0, 0, err + return 0, 0, 0, 0, err } start = uint32(val) } else { start = uint32(time.Now().Add(-time.Hour * 24 * 7).Unix()) } - return start, end, limit, nil + return start, end, limit, page, nil } func getAssetMaps(name string) (map[util.Uint256]*result.AssetUTXO, map[util.Uint256]*result.AssetUTXO, error) { @@ -533,7 +543,7 @@ func (s *Server) getUTXOTransfers(ps request.Params) (interface{}, *response.Err index++ } - start, end, limit, err := getTimestampsAndLimit(ps, index) + start, end, limit, page, err := getTimestampsAndLimit(ps, index) if err != nil { return nil, response.NewInvalidParamsError("", err) } @@ -543,21 +553,23 @@ func (s *Server) getUTXOTransfers(ps request.Params) (interface{}, *response.Err return nil, response.NewInvalidParamsError("", err) } tr := new(state.Transfer) + var resCount, frameCount int err = s.chain.ForEachTransfer(addr, tr, func() (bool, error) { + // Iterating from newest to oldest, not yet reached required + // time frame, continue looping. if tr.Timestamp > end { return true, nil } - var count int - for _, res := range sent { - count += len(res.Transactions) - } - for _, res := range recv { - count += len(res.Transactions) - } - if tr.Timestamp < start || - (limit != 0 && count >= limit) { + // Iterating from newest to oldest, moved past required + // time frame, stop looping. + if tr.Timestamp < start { return false, nil } + frameCount++ + // Using limits, not yet reached required page. + if limit != 0 && page*limit >= frameCount { + return true, nil + } assetID := core.GoverningTokenID() if !tr.IsGoverning { assetID = core.UtilityTokenID() @@ -576,6 +588,11 @@ func (s *Server) getUTXOTransfers(ps request.Params) (interface{}, *response.Err }) a.TotalAmount += tr.Amount } + resCount++ + // Using limits, reached limit. + if limit != 0 && resCount >= limit { + return false, nil + } return true, nil }) if err != nil { @@ -741,7 +758,7 @@ func (s *Server) getNEP5Transfers(ps request.Params) (interface{}, *response.Err return nil, response.ErrInvalidParams } - start, end, limit, err := getTimestampsAndLimit(ps, 1) + start, end, limit, page, err := getTimestampsAndLimit(ps, 1) if err != nil { return nil, response.NewInvalidParamsError("", err) } @@ -752,14 +769,23 @@ func (s *Server) getNEP5Transfers(ps request.Params) (interface{}, *response.Err Sent: []result.NEP5Transfer{}, } tr := new(state.NEP5Transfer) + var resCount, frameCount int err = s.chain.ForEachNEP5Transfer(u, tr, func() (bool, error) { + // Iterating from newest to oldest, not yet reached required + // time frame, continue looping. if tr.Timestamp > end { return true, nil } - if tr.Timestamp < start || - (limit != 0 && (len(bs.Received)+len(bs.Sent) >= limit)) { + // Iterating from newest to oldest, moved past required + // time frame, stop looping. + if tr.Timestamp < start { return false, nil } + frameCount++ + // Using limits, not yet reached required page. + if limit != 0 && page*limit >= frameCount { + return true, nil + } transfer := result.NEP5Transfer{ Timestamp: tr.Timestamp, Asset: tr.Asset, @@ -774,14 +800,18 @@ func (s *Server) getNEP5Transfers(ps request.Params) (interface{}, *response.Err transfer.Address = address.Uint160ToString(tr.From) } bs.Received = append(bs.Received, transfer) - return true, nil + } else { + transfer.Amount = strconv.FormatInt(-tr.Amount, 10) + if !tr.To.Equals(util.Uint160{}) { + transfer.Address = address.Uint160ToString(tr.To) + } + bs.Sent = append(bs.Sent, transfer) } - - transfer.Amount = strconv.FormatInt(-tr.Amount, 10) - if !tr.To.Equals(util.Uint160{}) { - transfer.Address = address.Uint160ToString(tr.To) + resCount++ + // Using limits, reached limit. + if limit != 0 && resCount >= limit { + return false, nil } - bs.Sent = append(bs.Sent, transfer) return true, nil }) if err != nil { diff --git a/pkg/rpc/server/server_test.go b/pkg/rpc/server/server_test.go index ce08abaeb..52f2487f6 100644 --- a/pkg/rpc/server/server_test.go +++ b/pkg/rpc/server/server_test.go @@ -1164,24 +1164,28 @@ func testRPCProtocol(t *testing.T, doRPCCall func(string, string, *testing.T) [] }) t.Run("getutxotransfers", func(t *testing.T) { - testGetUTXO := func(t *testing.T, asset string, start, stop int) { + testGetUTXO := func(t *testing.T, asset string, start, stop, limit, page int, present []int) { ps := []string{`"AKkkumHbBipZ46UMZJoFynJMXzSRnBvKcs"`} if asset != "" { ps = append(ps, fmt.Sprintf("%q", asset)) } - if start >= 0 { - if start > int(e.chain.HeaderHeight()) { - ps = append(ps, strconv.Itoa(int(time.Now().Unix()))) - } else { - b, err := e.chain.GetHeader(e.chain.GetHeaderHash(start)) - require.NoError(t, err) - ps = append(ps, strconv.Itoa(int(b.Timestamp))) - } - if stop != 0 { - b, err := e.chain.GetHeader(e.chain.GetHeaderHash(stop)) - require.NoError(t, err) - ps = append(ps, strconv.Itoa(int(b.Timestamp))) - } + if start > int(e.chain.HeaderHeight()) { + ps = append(ps, strconv.Itoa(int(time.Now().Unix()))) + } else { + b, err := e.chain.GetHeader(e.chain.GetHeaderHash(start)) + require.NoError(t, err) + ps = append(ps, strconv.Itoa(int(b.Timestamp))) + } + if stop != 0 { + b, err := e.chain.GetHeader(e.chain.GetHeaderHash(stop)) + require.NoError(t, err) + ps = append(ps, strconv.Itoa(int(b.Timestamp))) + } + if limit != 0 { + ps = append(ps, strconv.Itoa(limit)) + } + if page != 0 { + ps = append(ps, strconv.Itoa(page)) } p := strings.Join(ps, ", ") rpc := fmt.Sprintf(`{"jsonrpc": "2.0", "id": 1, "method": "getutxotransfers", "params": [%s]}`, p) @@ -1189,11 +1193,17 @@ func testRPCProtocol(t *testing.T, doRPCCall func(string, string, *testing.T) [] res := checkErrGetResult(t, body, false) actual := new(result.GetUTXO) require.NoError(t, json.Unmarshal(res, actual)) - checkTransfers(t, e, actual, asset, start, stop) + checkTransfers(t, e, actual, present) } - t.Run("RestrictByAsset", func(t *testing.T) { testGetUTXO(t, "neo", 0, 0) }) - t.Run("TooBigStart", func(t *testing.T) { testGetUTXO(t, "", 300, 0) }) - t.Run("RestrictAll", func(t *testing.T) { testGetUTXO(t, "", 202, 203) }) + // See `checkTransfers` for the last parameter values. + t.Run("All", func(t *testing.T) { testGetUTXO(t, "", 0, 207, 0, 0, []int{0, 1, 2, 3, 4, 5, 6, 7}) }) + t.Run("RestrictByAsset", func(t *testing.T) { testGetUTXO(t, "neo", 0, 0, 0, 0, []int{0, 1, 2, 6, 7}) }) + t.Run("TooBigStart", func(t *testing.T) { testGetUTXO(t, "", 300, 0, 0, 0, []int{}) }) + t.Run("RestrictAll", func(t *testing.T) { testGetUTXO(t, "", 202, 203, 0, 0, []int{1, 2, 3}) }) + t.Run("Limit", func(t *testing.T) { testGetUTXO(t, "neo", 0, 207, 2, 0, []int{7, 6}) }) + t.Run("Limit 2", func(t *testing.T) { testGetUTXO(t, "", 0, 204, 1, 0, []int{5}) }) + t.Run("Limit with page", func(t *testing.T) { testGetUTXO(t, "", 0, 204, 1, 1, []int{4}) }) + t.Run("Limit with page 2", func(t *testing.T) { testGetUTXO(t, "", 0, 204, 2, 2, []int{1, 0}) }) }) t.Run("getnep5transfers", func(t *testing.T) { @@ -1312,43 +1322,43 @@ func checkNep5TransfersAux(t *testing.T, e *executor, acc interface{}, onlyFirst require.Equal(t, uint32(0), res.Sent[0].NotifyIndex) } -func checkTransfers(t *testing.T, e *executor, acc interface{}, asset string, start, stop int) { +func checkTransfers(t *testing.T, e *executor, acc interface{}, checked []int) { + type transfer struct { + sent bool + asset string + index uint32 + amount int64 + } + + var transfers = []transfer{ + {false, "neo", 1, 99999000}, // NEO to us. + {false, "neo", 202, 99999000}, // NEO roundtrip for GAS claim. + {true, "neo", 202, 99999000}, // NEO roundtrip for GAS claim. + {false, "gas", 203, 160798392000}, // GAS claim. + {false, "gas", 204, 150798392000}, // Remainder from contract deployment. + {true, "gas", 204, 160798392000}, // Contract deployment. + {false, "neo", 206, 99998000}, // Remainder of NEO sent. + {true, "neo", 206, 99999000}, // NEO to another validator. + } res := acc.(*result.GetUTXO) require.Equal(t, res.Address, "AKkkumHbBipZ46UMZJoFynJMXzSRnBvKcs") - // transfer from multisig address to us - u := getUTXOForBlock(res, false, "neo", 1) - if start <= 1 && (stop == 0 || stop >= 1) && (asset == "neo" || asset == "") { - require.NotNil(t, u) - require.EqualValues(t, int64(99999000), u.Amount) - } else { - require.Nil(t, u) - } + for i, tr := range transfers { + var present bool - // gas claim - u = getUTXOForBlock(res, false, "gas", 203) - if start <= 203 && (stop == 0 || stop >= 203) && (asset == "gas" || asset == "") { - require.NotNil(t, u) - require.EqualValues(t, int64(160798392000), u.Amount) - } else { - require.Nil(t, u) - } - - // transfer from us to another validator - u = getUTXOForBlock(res, true, "neo", 206) - if start <= 206 && (stop == 0 || stop >= 206) && (asset == "neo" || asset == "") { - require.NotNil(t, u) - require.EqualValues(t, int64(99999000), u.Amount) - } else { - require.Nil(t, u) - } - - u = getUTXOForBlock(res, false, "neo", 206) - if start <= 206 && (stop == 0 || stop >= 206) && (asset == "neo" || asset == "") { - require.NotNil(t, u) - require.EqualValues(t, int64(99998000), u.Amount) - } else { - require.Nil(t, u) + u := getUTXOForBlock(res, tr.sent, tr.asset, tr.index) + for j := range checked { + if checked[j] == i { + present = true + break + } + } + if present { + require.NotNil(t, u) + require.EqualValues(t, tr.amount, u.Amount) + } else { + require.Nil(t, u) + } } }