forked from TrueCloudLab/rclone
operations: remove partially uploaded files on exit when not using --inplace
Before this change partially uploaded files (when --inplace is not in effect) would be left lying around in the file system if rclone was killed in the middle of a transfer. This adds an exit handler to remove the file and removes it when the file is complete.
This commit is contained in:
parent
7842000f8a
commit
6486ba6344
1 changed files with 26 additions and 0 deletions
|
@ -297,6 +297,20 @@ func removeFailedCopy(ctx context.Context, dst fs.Object) bool {
|
||||||
return true
|
return true
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Used to remove a failed partial copy
|
||||||
|
//
|
||||||
|
// Returns whether the file was successfully removed or not
|
||||||
|
func removeFailedPartialCopy(ctx context.Context, f fs.Fs, remotePartial string) bool {
|
||||||
|
o, err := f.NewObject(ctx, remotePartial)
|
||||||
|
if errors.Is(err, fs.ErrorObjectNotFound) {
|
||||||
|
return true
|
||||||
|
} else if err != nil {
|
||||||
|
fs.Infof(remotePartial, "Failed to remove failed partial copy: %s", err)
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
return removeFailedCopy(ctx, o)
|
||||||
|
}
|
||||||
|
|
||||||
// CommonHash returns a single hash.Type and a HashOption with that
|
// CommonHash returns a single hash.Type and a HashOption with that
|
||||||
// type which is in common between the two fs.Fs.
|
// type which is in common between the two fs.Fs.
|
||||||
func CommonHash(ctx context.Context, fa, fb fs.Info) (hash.Type, *fs.HashesOption) {
|
func CommonHash(ctx context.Context, fa, fb fs.Info) (hash.Type, *fs.HashesOption) {
|
||||||
|
@ -396,6 +410,14 @@ func Copy(ctx context.Context, f fs.Fs, dst fs.Object, remote string, src fs.Obj
|
||||||
}
|
}
|
||||||
// If can't server-side copy, do it manually
|
// If can't server-side copy, do it manually
|
||||||
if errors.Is(err, fs.ErrorCantCopy) {
|
if errors.Is(err, fs.ErrorCantCopy) {
|
||||||
|
// Remove partial files on premature exit
|
||||||
|
var atexitRemovePartial atexit.FnHandle
|
||||||
|
if !inplace {
|
||||||
|
atexitRemovePartial = atexit.Register(func() {
|
||||||
|
ctx := context.Background()
|
||||||
|
removeFailedPartialCopy(ctx, f, remotePartial)
|
||||||
|
})
|
||||||
|
}
|
||||||
if doMultiThreadCopy(ctx, f, src) {
|
if doMultiThreadCopy(ctx, f, src) {
|
||||||
// Number of streams proportional to size
|
// Number of streams proportional to size
|
||||||
streams := src.Size() / int64(ci.MultiThreadCutoff)
|
streams := src.Size() / int64(ci.MultiThreadCutoff)
|
||||||
|
@ -475,6 +497,10 @@ func Copy(ctx context.Context, f fs.Fs, dst fs.Object, remote string, src fs.Obj
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
if !inplace {
|
||||||
|
atexit.Unregister(atexitRemovePartial)
|
||||||
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
tries++
|
tries++
|
||||||
if tries >= maxTries {
|
if tries >= maxTries {
|
||||||
|
|
Loading…
Add table
Reference in a new issue