KurlesHS/rclone
1
0
Fork 0
forked from TrueCloudLab/rclone
Code Pull requests Activity

box: make listings of heavily used directories more reliable #5545

Browse source 
Before this change we uses limit/offset paging for directories in the
main directory listing routine and in the trash cleanup listing.

This switches to the new scheme of limit/marker which is more reliable
on a directory which is continuously changing. It has the disadvantage
that it doesn't tell us the total number of items available, however
that wasn't information rclone uses.
This commit is contained in:
Nick Craig-Wood 2021-08-20 10:13:33 +01:00
parent fc5d6c16b6
commit 308323e9c4
2 changed files with 19 additions and 12 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
backend/box/api/types.go
View file

@ -107,6 +107,7 @@ type FolderItems struct {
Entries []Item `json:"entries"` Entries []Item `json:"entries"`
Offset int `json:"offset"` Offset int `json:"offset"`
Limit int `json:"limit"` Limit int `json:"limit"`
NextMarker *string `json:"next_marker,omitempty"`
Order []struct { Order []struct {
By string `json:"by"` By string `json:"by"`
Direction string `json:"direction"` Direction string `json:"direction"`

22
backend/box/box.go
View file

@ -578,10 +578,13 @@ func (f *Fs) listAll(ctx context.Context, dirID string, directoriesOnly bool, fi
Parameters: fieldsValue(), Parameters: fieldsValue(),
} }
opts.Parameters.Set("limit", strconv.Itoa(listChunks)) opts.Parameters.Set("limit", strconv.Itoa(listChunks))
offset := 0 opts.Parameters.Set("usemarker", "true")
var marker *string
OUTER: OUTER:
for { for {
opts.Parameters.Set("offset", strconv.Itoa(offset)) if marker != nil {
opts.Parameters.Set("marker", *marker)
}
var result api.FolderItems var result api.FolderItems
var resp *http.Response var resp *http.Response
@ -615,8 +618,8 @@ OUTER:
break OUTER break OUTER
} }
} }
offset += result.Limit marker = result.NextMarker
if offset >= result.TotalCount { if marker == nil {
break break
} }
} }
@ -1100,9 +1103,12 @@ func (f *Fs) CleanUp(ctx context.Context) (err error) {
}, },
} }
opts.Parameters.Set("limit", strconv.Itoa(listChunks)) opts.Parameters.Set("limit", strconv.Itoa(listChunks))
offset := 0 opts.Parameters.Set("usemarker", "true")
var marker *string
for { for {
opts.Parameters.Set("offset", strconv.Itoa(offset)) if marker != nil {
opts.Parameters.Set("marker", *marker)
}
var result api.FolderItems var result api.FolderItems
var resp *http.Response var resp *http.Response
@ -1125,8 +1131,8 @@ func (f *Fs) CleanUp(ctx context.Context) (err error) {
continue continue
} }
} }
offset += result.Limit marker = result.NextMarker
if offset >= result.TotalCount { if marker == nil {
break break
} }
} }