From 6071db565c576a643faf10c560e756d803af8af9 Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Wed, 31 Mar 2021 09:52:47 +0100 Subject: [PATCH] onedrive: work around duplicated directory listing entries When paging big directories onedrive sometimes duplicates the last item on one page as the first item of the next page. This patch detects that and skips the duplicated item with an error message. See: https://forum.rclone.org/t/unexpected-duplicates-on-onedrive-with-0s-in-filename/23164 --- backend/onedrive/onedrive.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/backend/onedrive/onedrive.go b/backend/onedrive/onedrive.go index 17e505a60..18ab34118 100755 --- a/backend/onedrive/onedrive.go +++ b/backend/onedrive/onedrive.go @@ -897,6 +897,7 @@ func (f *Fs) listAll(ctx context.Context, dirID string, directoriesOnly bool, fi // Top parameter asks for bigger pages of data // https://dev.onedrive.com/odata/optional-query-parameters.htm opts := f.newOptsCall(dirID, "GET", "/children?$top=1000") + lastID := "\x00" OUTER: for { var result api.ListChildrenResponse @@ -911,6 +912,10 @@ OUTER: if len(result.Value) == 0 { break } + if result.Value[0].ID == lastID { + fs.Errorf(f, "Skipping duplicate entry %q in directory %q", lastID, dirID) + result.Value = result.Value[1:] + } for i := range result.Value { item := &result.Value[i] isFolder := item.GetFolder() != nil @@ -937,6 +942,9 @@ OUTER: } opts.Path = "" opts.RootURL = result.NextLink + if len(result.Value) > 0 { + lastID = result.Value[len(result.Value)-1].ID + } } return }