From 8eb16ce89cd9a9175587c1d1524bdc046f5922fe Mon Sep 17 00:00:00 2001
From: Nick Craig-Wood <nick@craig-wood.com>
Date: Mon, 3 Aug 2020 20:45:37 +0100
Subject: [PATCH] onedrive: implement rclone cleanup #4106
---
backend/onedrive/api/types.go | 25 ++++++++++++
backend/onedrive/onedrive.go | 71 +++++++++++++++++++++++++++++++++++
docs/content/onedrive.md | 14 +++++++
docs/content/overview.md | 2 +-
4 files changed, 111 insertions(+), 1 deletion(-)
diff --git a/backend/onedrive/api/types.go b/backend/onedrive/api/types.go
index 346829afa..53140819c 100644
--- a/backend/onedrive/api/types.go
+++ b/backend/onedrive/api/types.go
@@ -410,3 +410,28 @@ func (i *Item) GetParentReference() *ItemReference {
func (i *Item) IsRemote() bool {
return i.RemoteItem != nil
}
+
+// User details for each version
+type User struct {
+ Email string `json:"email"`
+ ID string `json:"id"`
+ DisplayName string `json:"displayName"`
+}
+
+// LastModifiedBy for each version
+type LastModifiedBy struct {
+ User User `json:"user"`
+}
+
+// Version info
+type Version struct {
+ ID string `json:"id"`
+ LastModifiedDateTime time.Time `json:"lastModifiedDateTime"`
+ Size int `json:"size"`
+ LastModifiedBy LastModifiedBy `json:"lastModifiedBy"`
+}
+
+// VersionsResponse is returned from /versions
+type VersionsResponse struct {
+ Versions []Version `json:"value"`
+}
diff --git a/backend/onedrive/onedrive.go b/backend/onedrive/onedrive.go
index ea4007b79..13cb10aa0 100755
--- a/backend/onedrive/onedrive.go
+++ b/backend/onedrive/onedrive.go
@@ -14,6 +14,7 @@ import (
"path"
"strconv"
"strings"
+ "sync"
"time"
"github.com/pkg/errors"
@@ -26,6 +27,8 @@ import (
"github.com/rclone/rclone/fs/config/obscure"
"github.com/rclone/rclone/fs/fserrors"
"github.com/rclone/rclone/fs/hash"
+ "github.com/rclone/rclone/fs/operations"
+ "github.com/rclone/rclone/fs/walk"
"github.com/rclone/rclone/lib/atexit"
"github.com/rclone/rclone/lib/dircache"
"github.com/rclone/rclone/lib/encoder"
@@ -1275,6 +1278,73 @@ func (f *Fs) PublicLink(ctx context.Context, remote string, expire fs.Duration,
return result.Link.WebURL, nil
}
+// CleanUp deletes all the hidden files.
+func (f *Fs) CleanUp(ctx context.Context) error {
+ token := make(chan struct{}, fs.Config.Checkers)
+ var wg sync.WaitGroup
+ err := walk.Walk(ctx, f, "", true, -1, func(path string, entries fs.DirEntries, err error) error {
+ err = entries.ForObjectError(func(obj fs.Object) error {
+ o, ok := obj.(*Object)
+ if !ok {
+ return errors.New("internal error: not a onedrive object")
+ }
+ wg.Add(1)
+ token <- struct{}{}
+ go func() {
+ defer func() {
+ <-token
+ wg.Done()
+ }()
+ err := o.deleteVersions(ctx)
+ if err != nil {
+ fs.Errorf(o, "Failed to remove versions: %v", err)
+ }
+ }()
+ return nil
+ })
+ wg.Wait()
+ return err
+ })
+ return err
+}
+
+// Finds and removes any old versions for o
+func (o *Object) deleteVersions(ctx context.Context) error {
+ opts := newOptsCall(o.id, "GET", "/versions")
+ var versions api.VersionsResponse
+ err := o.fs.pacer.Call(func() (bool, error) {
+ resp, err := o.fs.srv.CallJSON(ctx, &opts, nil, &versions)
+ return shouldRetry(resp, err)
+ })
+ if err != nil {
+ return err
+ }
+ if len(versions.Versions) < 2 {
+ return nil
+ }
+ for _, version := range versions.Versions[1:] {
+ err = o.deleteVersion(ctx, version.ID)
+ if err != nil {
+ return err
+ }
+ }
+ return nil
+}
+
+// Finds and removes any old versions for o
+func (o *Object) deleteVersion(ctx context.Context, ID string) error {
+ if operations.SkipDestructive(ctx, fmt.Sprintf("%s of %s", ID, o.remote), "delete version") {
+ return nil
+ }
+ fs.Infof(o, "removing version %q", ID)
+ opts := newOptsCall(o.id, "DELETE", "/versions/"+ID)
+ opts.NoResponse = true
+ return o.fs.pacer.Call(func() (bool, error) {
+ resp, err := o.fs.srv.Call(ctx, &opts)
+ return shouldRetry(resp, err)
+ })
+}
+
// ------------------------------------------------------------
// Fs returns the parent Fs
@@ -1840,6 +1910,7 @@ var (
_ fs.DirCacheFlusher = (*Fs)(nil)
_ fs.Abouter = (*Fs)(nil)
_ fs.PublicLinker = (*Fs)(nil)
+ _ fs.CleanUpper = (*Fs)(nil)
_ fs.Object = (*Object)(nil)
_ fs.MimeTyper = &Object{}
_ fs.IDer = &Object{}
diff --git a/docs/content/onedrive.md b/docs/content/onedrive.md
index 8a6e3d12d..ab91acb18 100644
--- a/docs/content/onedrive.md
+++ b/docs/content/onedrive.md
@@ -331,6 +331,8 @@ This counts against a users quota.
For example changing the modification time of a file creates a second
version, so the file is using twice the space.
+You can use the `rclone cleanup` command (see below) to remove old versions.
+
The `copy` is the only rclone command affected by this as we copy
the file and then afterwards set the modification time to match the
source file.
@@ -359,6 +361,18 @@ Note: This will disable the creation of new file versions, but will not remove a
8. Use rclone to upload or modify files. (I also use the --no-update-modtime flag)
9. Restore the versioning settings after using rclone. (Optional)
+### Cleanup
+
+OneDrive supports `rclone cleanup` which causes rclone to look through
+every file under the path supplied and delete all version but the
+current version. Because this involves traversing all the files, then
+querying each file for versions it can be quite slow. Rclone does
+`--checkers` tests in parallel. The command also supports `-i` which
+is a great way to see what it would do.
+
+ rclone cleanup -i remote:path/subdir # interactively remove all old version for path/subdir
+ rclone cleanup remote:path/subdir # unconditionally remove all old version for path/subdir
+
### Troubleshooting ###
#### Unexpected file size/hash differences on Sharepoint ####
diff --git a/docs/content/overview.md b/docs/content/overview.md
index 5cc94d156..f45a3bd88 100644
--- a/docs/content/overview.md
+++ b/docs/content/overview.md
@@ -336,7 +336,7 @@ operations more efficient.
| Mega | Yes | No | Yes | Yes | Yes | No | No | No [#2178](https://github.com/rclone/rclone/issues/2178) | Yes | Yes |
| Memory | No | Yes | No | No | No | Yes | Yes | No | No | No |
| Microsoft Azure Blob Storage | Yes | Yes | No | No | No | Yes | Yes | No [#2178](https://github.com/rclone/rclone/issues/2178) | No | No |
-| Microsoft OneDrive | Yes | Yes | Yes | Yes | No [#575](https://github.com/rclone/rclone/issues/575) | No | No | Yes | Yes | Yes |
+| Microsoft OneDrive | Yes | Yes | Yes | Yes | Yes | No | No | Yes | Yes | Yes |
| OpenDrive | Yes | Yes | Yes | Yes | No | No | No | No | No | Yes |
| OpenStack Swift | Yes † | Yes | No | No | No | Yes | Yes | No [#2178](https://github.com/rclone/rclone/issues/2178) | Yes | No |
| pCloud | Yes | Yes | Yes | Yes | Yes | No | No | Yes | Yes | Yes |