From 308323e9c4ca3d5d9d8226c8162b5f5df24e9b1a Mon Sep 17 00:00:00 2001
From: Nick Craig-Wood <nick@craig-wood.com>
Date: Fri, 20 Aug 2021 10:13:33 +0100
Subject: [PATCH] box: make listings of heavily used directories more reliable
#5545
Before this change we uses limit/offset paging for directories in the
main directory listing routine and in the trash cleanup listing.
This switches to the new scheme of limit/marker which is more reliable
on a directory which is continuously changing. It has the disadvantage
that it doesn't tell us the total number of items available, however
that wasn't information rclone uses.
---
backend/box/api/types.go | 9 +++++----
backend/box/box.go | 22 ++++++++++++++--------
2 files changed, 19 insertions(+), 12 deletions(-)
diff --git a/backend/box/api/types.go b/backend/box/api/types.go
index 68d6aac2a..363fa3353 100644
--- a/backend/box/api/types.go
+++ b/backend/box/api/types.go
@@ -103,10 +103,11 @@ func (i *Item) ModTime() (t time.Time) {
// FolderItems is returned from the GetFolderItems call
type FolderItems struct {
- TotalCount int `json:"total_count"`
- Entries []Item `json:"entries"`
- Offset int `json:"offset"`
- Limit int `json:"limit"`
+ TotalCount int `json:"total_count"`
+ Entries []Item `json:"entries"`
+ Offset int `json:"offset"`
+ Limit int `json:"limit"`
+ NextMarker *string `json:"next_marker,omitempty"`
Order []struct {
By string `json:"by"`
Direction string `json:"direction"`
diff --git a/backend/box/box.go b/backend/box/box.go
index 2af0a857e..7f41def85 100644
--- a/backend/box/box.go
+++ b/backend/box/box.go
@@ -578,10 +578,13 @@ func (f *Fs) listAll(ctx context.Context, dirID string, directoriesOnly bool, fi
Parameters: fieldsValue(),
}
opts.Parameters.Set("limit", strconv.Itoa(listChunks))
- offset := 0
+ opts.Parameters.Set("usemarker", "true")
+ var marker *string
OUTER:
for {
- opts.Parameters.Set("offset", strconv.Itoa(offset))
+ if marker != nil {
+ opts.Parameters.Set("marker", *marker)
+ }
var result api.FolderItems
var resp *http.Response
@@ -615,8 +618,8 @@ OUTER:
break OUTER
}
}
- offset += result.Limit
- if offset >= result.TotalCount {
+ marker = result.NextMarker
+ if marker == nil {
break
}
}
@@ -1100,9 +1103,12 @@ func (f *Fs) CleanUp(ctx context.Context) (err error) {
},
}
opts.Parameters.Set("limit", strconv.Itoa(listChunks))
- offset := 0
+ opts.Parameters.Set("usemarker", "true")
+ var marker *string
for {
- opts.Parameters.Set("offset", strconv.Itoa(offset))
+ if marker != nil {
+ opts.Parameters.Set("marker", *marker)
+ }
var result api.FolderItems
var resp *http.Response
@@ -1125,8 +1131,8 @@ func (f *Fs) CleanUp(ctx context.Context) (err error) {
continue
}
}
- offset += result.Limit
- if offset >= result.TotalCount {
+ marker = result.NextMarker
+ if marker == nil {
break
}
}