From 6486ba63446fb733cbbf6fc2b22b6d3010347888 Mon Sep 17 00:00:00 2001
From: Nick Craig-Wood <nick@craig-wood.com>
Date: Wed, 24 May 2023 11:15:24 +0100
Subject: [PATCH] operations: remove partially uploaded files on exit when not
using --inplace
Before this change partially uploaded files (when --inplace is not in
effect) would be left lying around in the file system if rclone was
killed in the middle of a transfer.
This adds an exit handler to remove the file and removes it when the
file is complete.
---
fs/operations/operations.go | 26 ++++++++++++++++++++++++++
1 file changed, 26 insertions(+)
diff --git a/fs/operations/operations.go b/fs/operations/operations.go
index 06ab1be4d..4308df7fa 100644
--- a/fs/operations/operations.go
+++ b/fs/operations/operations.go
@@ -297,6 +297,20 @@ func removeFailedCopy(ctx context.Context, dst fs.Object) bool {
return true
}
+// Used to remove a failed partial copy
+//
+// Returns whether the file was successfully removed or not
+func removeFailedPartialCopy(ctx context.Context, f fs.Fs, remotePartial string) bool {
+ o, err := f.NewObject(ctx, remotePartial)
+ if errors.Is(err, fs.ErrorObjectNotFound) {
+ return true
+ } else if err != nil {
+ fs.Infof(remotePartial, "Failed to remove failed partial copy: %s", err)
+ return false
+ }
+ return removeFailedCopy(ctx, o)
+}
+
// CommonHash returns a single hash.Type and a HashOption with that
// type which is in common between the two fs.Fs.
func CommonHash(ctx context.Context, fa, fb fs.Info) (hash.Type, *fs.HashesOption) {
@@ -396,6 +410,14 @@ func Copy(ctx context.Context, f fs.Fs, dst fs.Object, remote string, src fs.Obj
}
// If can't server-side copy, do it manually
if errors.Is(err, fs.ErrorCantCopy) {
+ // Remove partial files on premature exit
+ var atexitRemovePartial atexit.FnHandle
+ if !inplace {
+ atexitRemovePartial = atexit.Register(func() {
+ ctx := context.Background()
+ removeFailedPartialCopy(ctx, f, remotePartial)
+ })
+ }
if doMultiThreadCopy(ctx, f, src) {
// Number of streams proportional to size
streams := src.Size() / int64(ci.MultiThreadCutoff)
@@ -475,6 +497,10 @@ func Copy(ctx context.Context, f fs.Fs, dst fs.Object, remote string, src fs.Obj
}
}
}
+ if !inplace {
+ atexit.Unregister(atexitRemovePartial)
+ }
+
}
tries++
if tries >= maxTries {