From b89f8c05cfd760abb9b5c403255a66b16f057cf2 Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Fri, 11 Sep 2020 16:34:54 +0100 Subject: [PATCH] vfs: detect and recover from a file being removed externally from the cache Before this change if a file was removed from the cache while rclone is running then rclone would not notice and proceed to re-create it full of zeros. This change notices files that we expect to have data in going missing and if they do logs an ERROR recovers. It isn't recommended deleting files from the cache manually with rclone running! See: https://forum.rclone.org/t/corrupted-data-streaming-after-vfs-meta-files-removed/18997 Fixes #4602 --- vfs/vfscache/item.go | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/vfs/vfscache/item.go b/vfs/vfscache/item.go index e828b9cb8..5a997f076 100644 --- a/vfs/vfscache/item.go +++ b/vfs/vfscache/item.go @@ -239,8 +239,23 @@ func (item *Item) _truncate(size int64) (err error) { // Use open handle if available fd := item.fd if fd == nil { + // If the metadata says we have some blockes cached then the + // file should exist, so open without O_CREATE + oFlags := os.O_WRONLY + if item.info.Rs.Size() == 0 { + oFlags |= os.O_CREATE + } osPath := item.c.toOSPath(item.name) // No locking in Cache - fd, err = file.OpenFile(osPath, os.O_CREATE|os.O_WRONLY, 0600) + fd, err = file.OpenFile(osPath, oFlags, 0600) + if err != nil && os.IsNotExist(err) { + // If the metadata has info but the file doesn't + // not exist then it has been externally removed + fs.Errorf(item.name, "vfs cache: detected external removal of cache file") + item.info.Rs = nil // show we have no blocks cached + item.info.Dirty = false // file can't be dirty if it doesn't exist + item._removeMeta("cache file externally deleted") + fd, err = file.OpenFile(osPath, os.O_CREATE|os.O_WRONLY, 0600) + } if err != nil { return errors.Wrap(err, "vfs cache: truncate: failed to open cache file") }