From 308323e9c4ca3d5d9d8226c8162b5f5df24e9b1a Mon Sep 17 00:00:00 2001
From: Nick Craig-Wood <nick@craig-wood.com>
Date: Fri, 20 Aug 2021 10:13:33 +0100
Subject: [PATCH] box: make listings of heavily used directories more reliable
 #5545

Before this change we uses limit/offset paging for directories in the
main directory listing routine and in the trash cleanup listing.

This switches to the new scheme of limit/marker which is more reliable
on a directory which is continuously changing. It has the disadvantage
that it doesn't tell us the total number of items available, however
that wasn't information rclone uses.
---
 backend/box/api/types.go |  9 +++++----
 backend/box/box.go       | 22 ++++++++++++++--------
 2 files changed, 19 insertions(+), 12 deletions(-)

diff --git a/backend/box/api/types.go b/backend/box/api/types.go
index 68d6aac2a..363fa3353 100644
--- a/backend/box/api/types.go
+++ b/backend/box/api/types.go
@@ -103,10 +103,11 @@ func (i *Item) ModTime() (t time.Time) {
 
 // FolderItems is returned from the GetFolderItems call
 type FolderItems struct {
-	TotalCount int    `json:"total_count"`
-	Entries    []Item `json:"entries"`
-	Offset     int    `json:"offset"`
-	Limit      int    `json:"limit"`
+	TotalCount int     `json:"total_count"`
+	Entries    []Item  `json:"entries"`
+	Offset     int     `json:"offset"`
+	Limit      int     `json:"limit"`
+	NextMarker *string `json:"next_marker,omitempty"`
 	Order      []struct {
 		By        string `json:"by"`
 		Direction string `json:"direction"`
diff --git a/backend/box/box.go b/backend/box/box.go
index 2af0a857e..7f41def85 100644
--- a/backend/box/box.go
+++ b/backend/box/box.go
@@ -578,10 +578,13 @@ func (f *Fs) listAll(ctx context.Context, dirID string, directoriesOnly bool, fi
 		Parameters: fieldsValue(),
 	}
 	opts.Parameters.Set("limit", strconv.Itoa(listChunks))
-	offset := 0
+	opts.Parameters.Set("usemarker", "true")
+	var marker *string
 OUTER:
 	for {
-		opts.Parameters.Set("offset", strconv.Itoa(offset))
+		if marker != nil {
+			opts.Parameters.Set("marker", *marker)
+		}
 
 		var result api.FolderItems
 		var resp *http.Response
@@ -615,8 +618,8 @@ OUTER:
 				break OUTER
 			}
 		}
-		offset += result.Limit
-		if offset >= result.TotalCount {
+		marker = result.NextMarker
+		if marker == nil {
 			break
 		}
 	}
@@ -1100,9 +1103,12 @@ func (f *Fs) CleanUp(ctx context.Context) (err error) {
 		},
 	}
 	opts.Parameters.Set("limit", strconv.Itoa(listChunks))
-	offset := 0
+	opts.Parameters.Set("usemarker", "true")
+	var marker *string
 	for {
-		opts.Parameters.Set("offset", strconv.Itoa(offset))
+		if marker != nil {
+			opts.Parameters.Set("marker", *marker)
+		}
 
 		var result api.FolderItems
 		var resp *http.Response
@@ -1125,8 +1131,8 @@ func (f *Fs) CleanUp(ctx context.Context) (err error) {
 				continue
 			}
 		}
-		offset += result.Limit
-		if offset >= result.TotalCount {
+		marker = result.NextMarker
+		if marker == nil {
 			break
 		}
 	}