From 61c18e3b602b510e06a182924532e0f1ab4d2055 Mon Sep 17 00:00:00 2001
From: Nick Craig-Wood <nick@craig-wood.com>
Date: Tue, 11 Jun 2024 19:26:38 +0100
Subject: [PATCH] zoho: use cursor listing for improved performance

Cursor listing enables us to list up to 1,000 items per call
(previously it was 10) and uses one less transaction per call.

See: https://forum.rclone.org/t/second-followup-on-the-older-topic-rclone-invokes-more-number-of-workdrive-s-files-listing-api-calls-which-exceeds-the-throttling-limit/45697/4
---
 backend/zoho/api/types.go |  9 +++++++++
 backend/zoho/zoho.go      | 20 ++++++++++++++------
 2 files changed, 23 insertions(+), 6 deletions(-)

diff --git a/backend/zoho/api/types.go b/backend/zoho/api/types.go
index 2414f977c..5a516b6fb 100644
--- a/backend/zoho/api/types.go
+++ b/backend/zoho/api/types.go
@@ -70,8 +70,17 @@ type ItemInfo struct {
 	Item Item `json:"data"`
 }
 
+// Links contains Cursor information
+type Links struct {
+	Cursor struct {
+		HasNext bool   `json:"has_next"`
+		Next    string `json:"next"`
+	} `json:"cursor"`
+}
+
 // ItemList contains multiple Zoho Items
 type ItemList struct {
+	Links Links  `json:"links"`
 	Items []Item `json:"data"`
 }
 
diff --git a/backend/zoho/zoho.go b/backend/zoho/zoho.go
index f8ea5755c..0eabc279e 100644
--- a/backend/zoho/zoho.go
+++ b/backend/zoho/zoho.go
@@ -454,18 +454,18 @@ type listAllFn func(*api.Item) bool
 //
 // If the user fn ever returns true then it early exits with found = true
 func (f *Fs) listAll(ctx context.Context, dirID string, directoriesOnly bool, filesOnly bool, fn listAllFn) (found bool, err error) {
+	const listItemsLimit = 1000
 	opts := rest.Opts{
 		Method:       "GET",
 		Path:         "/files/" + dirID + "/files",
 		ExtraHeaders: map[string]string{"Accept": "application/vnd.api+json"},
-		Parameters:   url.Values{},
+		Parameters: url.Values{
+			"page[limit]": {strconv.Itoa(listItemsLimit)},
+			"page[next]":  {"0"},
+		},
 	}
-	opts.Parameters.Set("page[limit]", strconv.Itoa(10))
-	offset := 0
 OUTER:
 	for {
-		opts.Parameters.Set("page[offset]", strconv.Itoa(offset))
-
 		var result api.ItemList
 		var resp *http.Response
 		err = f.pacer.Call(func() (bool, error) {
@@ -495,7 +495,15 @@ OUTER:
 				break OUTER
 			}
 		}
-		offset += 10
+		if !result.Links.Cursor.HasNext {
+			break
+		}
+		// Fetch the next from the URL in the response
+		nextURL, err := url.Parse(result.Links.Cursor.Next)
+		if err != nil {
+			return found, fmt.Errorf("failed to parse next link as URL: %w", err)
+		}
+		opts.Parameters.Set("page[next]", nextURL.Query().Get("page[next]"))
 	}
 	return
 }