TrueCloudLab/rclone
16
0
Fork 2
Code Issues Pull requests 1 Projects Releases Packages Wiki Activity Actions

vfs: implement --vfs-cache-max-size to limit the total size of the cache

Browse source
This commit is contained in:
Nick Craig-Wood 2019-02-01 23:35:03 +00:00
parent fffdbb31f5
commit a43ed567ee
5 changed files with 367 additions and 105 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

125
vfs/cache.go
View file

@ -67,8 +67,9 @@ type cache struct {
f fs.Fs // fs for the cache directory f fs.Fs // fs for the cache directory
opt *Options // vfs Options opt *Options // vfs Options
root string // root of the cache directory root string // root of the cache directory
itemMu sync.Mutex // protects the next two maps itemMu sync.Mutex // protects the following variables
item map[string]*cacheItem // files/directories in the cache item map[string]*cacheItem // files/directories in the cache
used int64 // total size of files in the cache
} }
// cacheItem is stored in the item map // cacheItem is stored in the item map
@ -76,6 +77,7 @@ type cacheItem struct {
opens int // number of times file is open opens int // number of times file is open
atime time.Time // last time file was accessed atime time.Time // last time file was accessed
isFile bool // if this is a file or a directory isFile bool // if this is a file or a directory
size int64 // size of the cached item
} }
// newCacheItem returns an item for the cache // newCacheItem returns an item for the cache
@ -196,11 +198,13 @@ func (c *cache) get(name string) *cacheItem {
return item return item
} }
// updateTime sets the atime of the name to that passed in if it is // updateStat sets the atime of the name to that passed in if it is
// newer than the existing or there isn't an existing time. // newer than the existing or there isn't an existing time.
// //
// it also sets the size
//
// name should be a remote path not an osPath // name should be a remote path not an osPath
func (c *cache) updateTime(name string, when time.Time) { func (c *cache) updateStat(name string, when time.Time, size int64) {
name = clean(name) name = clean(name)
c.itemMu.Lock() c.itemMu.Lock()
item, found := c._get(true, name) item, found := c._get(true, name)
@ -208,6 +212,7 @@ func (c *cache) updateTime(name string, when time.Time) {
fs.Debugf(name, "updateTime: setting atime to %v", when) fs.Debugf(name, "updateTime: setting atime to %v", when)
item.atime = when item.atime = when
} }
item.size = size
c.itemMu.Unlock() c.itemMu.Unlock()
} }
@ -266,6 +271,12 @@ func (c *cache) _close(isFile bool, name string) {
if item.opens < 0 { if item.opens < 0 {
fs.Errorf(name, "cache: double close") fs.Errorf(name, "cache: double close")
} }
osPath := c.toOSPath(name)
fi, err := os.Stat(osPath)
// Update the size on close
if err == nil && !fi.IsDir() {
item.size = fi.Size()
}
if name == "" { if name == "" {
break break
} }
@ -291,7 +302,7 @@ func (c *cache) remove(name string) {
if err != nil && !os.IsNotExist(err) { if err != nil && !os.IsNotExist(err) {
fs.Errorf(name, "Failed to remove from cache: %v", err) fs.Errorf(name, "Failed to remove from cache: %v", err)
} else { } else {
fs.Debugf(name, "Removed from cache") fs.Infof(name, "Removed from cache")
} }
} }
@ -338,26 +349,34 @@ func (c *cache) walk(fn func(osPath string, fi os.FileInfo, name string) error)
}) })
} }
// updateAtimes walks the cache updating any atimes it finds // updateStats walks the cache updating any atimes and sizes it finds
func (c *cache) updateAtimes() error { //
return c.walk(func(osPath string, fi os.FileInfo, name string) error { // it also updates used
func (c *cache) updateStats() error {
var newUsed int64
err := c.walk(func(osPath string, fi os.FileInfo, name string) error {
if !fi.IsDir() { if !fi.IsDir() {
// Update the atime with that of the file // Update the atime with that of the file
atime := times.Get(fi).AccessTime() atime := times.Get(fi).AccessTime()
c.updateTime(name, atime) c.updateStat(name, atime, fi.Size())
newUsed += fi.Size()
} else { } else {
c.cacheDir(name) c.cacheDir(name)
} }
return nil return nil
}) })
c.itemMu.Lock()
c.used = newUsed
c.itemMu.Unlock()
return err
} }
// purgeOld gets rid of any files that are over age // purgeOld gets rid of any files that are over age
func (c *cache) purgeOld(maxAge time.Duration) { func (c *cache) purgeOld(maxAge time.Duration) {
c._purgeOld(maxAge, c.remove, c.removeDir) c._purgeOld(maxAge, c.remove)
} }
func (c *cache) _purgeOld(maxAge time.Duration, remove func(name string), removeDir func(name string) bool) { func (c *cache) _purgeOld(maxAge time.Duration, remove func(name string)) {
c.itemMu.Lock() c.itemMu.Lock()
defer c.itemMu.Unlock() defer c.itemMu.Unlock()
cutoff := time.Now().Add(-maxAge) cutoff := time.Now().Add(-maxAge)
@ -373,7 +392,16 @@ func (c *cache) _purgeOld(maxAge time.Duration, remove func(name string), remove
} }
} }
} }
// now find any empty directories }
// Purge any empty directories
func (c *cache) purgeEmptyDirs() {
c._purgeEmptyDirs(c.removeDir)
}
func (c *cache) _purgeEmptyDirs(removeDir func(name string) bool) {
c.itemMu.Lock()
defer c.itemMu.Unlock()
var dirs []string var dirs []string
for name, item := range c.item { for name, item := range c.item {
if !item.isFile && item.opens == 0 { if !item.isFile && item.opens == 0 {
@ -391,6 +419,56 @@ func (c *cache) _purgeOld(maxAge time.Duration, remove func(name string), remove
} }
} }
// This is a cacheItem with a name for sorting
type cacheNamedItem struct {
name string
item *cacheItem
}
type cacheNamedItems []cacheNamedItem
func (v cacheNamedItems) Len() int { return len(v) }
func (v cacheNamedItems) Swap(i, j int) { v[i], v[j] = v[j], v[i] }
func (v cacheNamedItems) Less(i, j int) bool { return v[i].item.atime.Before(v[j].item.atime) }
// Remove any files that are over quota starting from the
// oldest first
func (c *cache) purgeOverQuota(quota int64) {
c._purgeOverQuota(quota, c.remove)
}
func (c *cache) _purgeOverQuota(quota int64, remove func(name string)) {
c.itemMu.Lock()
defer c.itemMu.Unlock()
if quota <= 0 || c.used < quota {
return
}
var items cacheNamedItems
// Make a slice of unused files
for name, item := range c.item {
if item.isFile && item.opens == 0 {
items = append(items, cacheNamedItem{
name: name,
item: item,
})
}
}
sort.Sort(items)
// Remove items until the quota is OK
for _, item := range items {
if c.used < quota {
break
}
remove(item.name)
// Remove the entry
delete(c.item, item.name)
c.used -= item.item.size
}
}
// clean empties the cache of stuff if it can // clean empties the cache of stuff if it can
func (c *cache) clean() { func (c *cache) clean() {
// Cache may be empty so end // Cache may be empty so end
@ -399,17 +477,32 @@ func (c *cache) clean() {
return return
} }
fs.Debugf(nil, "Cleaning the cache") c.itemMu.Lock()
oldItems, oldUsed := len(c.item), fs.SizeSuffix(c.used)
c.itemMu.Unlock()
// first walk the FS to update the atimes // first walk the FS to update the atimes and sizes
err = c.updateAtimes() err = c.updateStats()
if err != nil { if err != nil {
fs.Errorf(nil, "Error traversing cache %q: %v", c.root, err) fs.Errorf(nil, "Error traversing cache %q: %v", c.root, err)
} }
// Now remove any files that are over age and any empty // Remove any files that are over age
// directories
c.purgeOld(c.opt.CacheMaxAge) c.purgeOld(c.opt.CacheMaxAge)
// Now remove any files that are over quota starting from the
// oldest first
c.purgeOverQuota(int64(c.opt.CacheMaxSize))
// Remove any empty directories
c.purgeEmptyDirs()
// Stats
c.itemMu.Lock()
newItems, newUsed := len(c.item), fs.SizeSuffix(c.used)
c.itemMu.Unlock()
fs.Infof(nil, "Cleaned the cache: objects %d (was %d), total size %v (was %v)", newItems, oldItems, newUsed, oldUsed)
} }
// cleaner calls clean at regular intervals // cleaner calls clean at regular intervals

338
vfs/cache_test.go
View file

@ -51,7 +51,7 @@ func itemAsString(c *cache) []string {
defer c.itemMu.Unlock() defer c.itemMu.Unlock()
var out []string var out []string
for name, item := range c.item { for name, item := range c.item {
out = append(out, fmt.Sprintf("name=%q isFile=%v opens=%d", name, item.isFile, item.opens)) out = append(out, fmt.Sprintf("name=%q isFile=%v opens=%d size=%d", name, item.isFile, item.opens, item.size))
} }
sort.Strings(out) sort.Strings(out)
return out return out
@ -79,7 +79,7 @@ func TestCacheNew(t *testing.T) {
require.NoError(t, err) require.NoError(t, err)
assert.Equal(t, "potato", filepath.Base(p)) assert.Equal(t, "potato", filepath.Base(p))
assert.Equal(t, []string{ assert.Equal(t, []string{
`name="" isFile=false opens=0`, `name="" isFile=false opens=0 size=0`,
}, itemAsString(c)) }, itemAsString(c))
fi, err := os.Stat(filepath.Dir(p)) fi, err := os.Stat(filepath.Dir(p))
@ -95,26 +95,26 @@ func TestCacheNew(t *testing.T) {
// updateTime // updateTime
//.. before //.. before
t1 := time.Now().Add(-60 * time.Minute) t1 := time.Now().Add(-60 * time.Minute)
c.updateTime("potato", t1) c.updateStat("potato", t1, 0)
item = c.get("potato") item = c.get("potato")
assert.NotEqual(t, t1, item.atime) assert.NotEqual(t, t1, item.atime)
assert.Equal(t, 0, item.opens) assert.Equal(t, 0, item.opens)
//..after //..after
t2 := time.Now().Add(60 * time.Minute) t2 := time.Now().Add(60 * time.Minute)
c.updateTime("potato", t2) c.updateStat("potato", t2, 0)
item = c.get("potato") item = c.get("potato")
assert.Equal(t, t2, item.atime) assert.Equal(t, t2, item.atime)
assert.Equal(t, 0, item.opens) assert.Equal(t, 0, item.opens)
// open // open
assert.Equal(t, []string{ assert.Equal(t, []string{
`name="" isFile=false opens=0`, `name="" isFile=false opens=0 size=0`,
`name="potato" isFile=true opens=0`, `name="potato" isFile=true opens=0 size=0`,
}, itemAsString(c)) }, itemAsString(c))
c.open("/potato") c.open("/potato")
assert.Equal(t, []string{ assert.Equal(t, []string{
`name="" isFile=false opens=1`, `name="" isFile=false opens=1 size=0`,
`name="potato" isFile=true opens=1`, `name="potato" isFile=true opens=1 size=0`,
}, itemAsString(c)) }, itemAsString(c))
item = c.get("potato") item = c.get("potato")
assert.WithinDuration(t, time.Now(), item.atime, time.Second) assert.WithinDuration(t, time.Now(), item.atime, time.Second)
@ -132,11 +132,11 @@ func TestCacheNew(t *testing.T) {
// updateAtimes // updateAtimes
item = c.get("potato") item = c.get("potato")
item.atime = time.Now().Add(-24 * time.Hour) item.atime = time.Now().Add(-24 * time.Hour)
err = c.updateAtimes() err = c.updateStats()
require.NoError(t, err) require.NoError(t, err)
assert.Equal(t, []string{ assert.Equal(t, []string{
`name="" isFile=false opens=1`, `name="" isFile=false opens=1 size=0`,
`name="potato" isFile=true opens=1`, `name="potato" isFile=true opens=1 size=5`,
}, itemAsString(c)) }, itemAsString(c))
item = c.get("potato") item = c.get("potato")
assert.Equal(t, atime, item.atime) assert.Equal(t, atime, item.atime)
@ -146,11 +146,11 @@ func TestCacheNew(t *testing.T) {
c.itemMu.Lock() c.itemMu.Lock()
delete(c.item, "potato") // remove from cache delete(c.item, "potato") // remove from cache
c.itemMu.Unlock() c.itemMu.Unlock()
err = c.updateAtimes() err = c.updateStats()
require.NoError(t, err) require.NoError(t, err)
assert.Equal(t, []string{ assert.Equal(t, []string{
`name="" isFile=false opens=1`, `name="" isFile=false opens=1 size=0`,
`name="potato" isFile=true opens=0`, `name="potato" isFile=true opens=0 size=5`,
}, itemAsString(c)) }, itemAsString(c))
item = c.get("potato") item = c.get("potato")
assert.Equal(t, atime, item.atime) assert.Equal(t, atime, item.atime)
@ -165,18 +165,18 @@ func TestCacheNew(t *testing.T) {
// close // close
assert.Equal(t, []string{ assert.Equal(t, []string{
`name="" isFile=false opens=1`, `name="" isFile=false opens=1 size=0`,
`name="potato" isFile=true opens=1`, `name="potato" isFile=true opens=1 size=5`,
}, itemAsString(c)) }, itemAsString(c))
c.updateTime("potato", t2) c.updateStat("potato", t2, 6)
assert.Equal(t, []string{ assert.Equal(t, []string{
`name="" isFile=false opens=1`, `name="" isFile=false opens=1 size=0`,
`name="potato" isFile=true opens=1`, `name="potato" isFile=true opens=1 size=6`,
}, itemAsString(c)) }, itemAsString(c))
c.close("potato/") c.close("potato/")
assert.Equal(t, []string{ assert.Equal(t, []string{
`name="" isFile=false opens=0`, `name="" isFile=false opens=0 size=0`,
`name="potato" isFile=true opens=0`, `name="potato" isFile=true opens=0 size=5`,
}, itemAsString(c)) }, itemAsString(c))
item = c.get("potato") item = c.get("potato")
assert.WithinDuration(t, time.Now(), item.atime, time.Second) assert.WithinDuration(t, time.Now(), item.atime, time.Second)
@ -216,23 +216,23 @@ func TestCacheOpens(t *testing.T) {
assert.Equal(t, []string(nil), itemAsString(c)) assert.Equal(t, []string(nil), itemAsString(c))
c.open("potato") c.open("potato")
assert.Equal(t, []string{ assert.Equal(t, []string{
`name="" isFile=false opens=1`, `name="" isFile=false opens=1 size=0`,
`name="potato" isFile=true opens=1`, `name="potato" isFile=true opens=1 size=0`,
}, itemAsString(c)) }, itemAsString(c))
c.open("potato") c.open("potato")
assert.Equal(t, []string{ assert.Equal(t, []string{
`name="" isFile=false opens=2`, `name="" isFile=false opens=2 size=0`,
`name="potato" isFile=true opens=2`, `name="potato" isFile=true opens=2 size=0`,
}, itemAsString(c)) }, itemAsString(c))
c.close("potato") c.close("potato")
assert.Equal(t, []string{ assert.Equal(t, []string{
`name="" isFile=false opens=1`, `name="" isFile=false opens=1 size=0`,
`name="potato" isFile=true opens=1`, `name="potato" isFile=true opens=1 size=0`,
}, itemAsString(c)) }, itemAsString(c))
c.close("potato") c.close("potato")
assert.Equal(t, []string{ assert.Equal(t, []string{
`name="" isFile=false opens=0`, `name="" isFile=false opens=0 size=0`,
`name="potato" isFile=true opens=0`, `name="potato" isFile=true opens=0 size=0`,
}, itemAsString(c)) }, itemAsString(c))
c.open("potato") c.open("potato")
@ -240,34 +240,34 @@ func TestCacheOpens(t *testing.T) {
c.open("a/b/c/d/e/two") c.open("a/b/c/d/e/two")
c.open("a/b/c/d/e/f/three") c.open("a/b/c/d/e/f/three")
assert.Equal(t, []string{ assert.Equal(t, []string{
`name="" isFile=false opens=4`, `name="" isFile=false opens=4 size=0`,
`name="a" isFile=false opens=3`, `name="a" isFile=false opens=3 size=0`,
`name="a/b" isFile=false opens=3`, `name="a/b" isFile=false opens=3 size=0`,
`name="a/b/c" isFile=false opens=3`, `name="a/b/c" isFile=false opens=3 size=0`,
`name="a/b/c/d" isFile=false opens=3`, `name="a/b/c/d" isFile=false opens=3 size=0`,
`name="a/b/c/d/e" isFile=false opens=2`, `name="a/b/c/d/e" isFile=false opens=2 size=0`,
`name="a/b/c/d/e/f" isFile=false opens=1`, `name="a/b/c/d/e/f" isFile=false opens=1 size=0`,
`name="a/b/c/d/e/f/three" isFile=true opens=1`, `name="a/b/c/d/e/f/three" isFile=true opens=1 size=0`,
`name="a/b/c/d/e/two" isFile=true opens=1`, `name="a/b/c/d/e/two" isFile=true opens=1 size=0`,
`name="a/b/c/d/one" isFile=true opens=1`, `name="a/b/c/d/one" isFile=true opens=1 size=0`,
`name="potato" isFile=true opens=1`, `name="potato" isFile=true opens=1 size=0`,
}, itemAsString(c)) }, itemAsString(c))
c.close("potato") c.close("potato")
c.close("a/b/c/d/one") c.close("a/b/c/d/one")
c.close("a/b/c/d/e/two") c.close("a/b/c/d/e/two")
c.close("a/b/c//d/e/f/three") c.close("a/b/c//d/e/f/three")
assert.Equal(t, []string{ assert.Equal(t, []string{
`name="" isFile=false opens=0`, `name="" isFile=false opens=0 size=0`,
`name="a" isFile=false opens=0`, `name="a" isFile=false opens=0 size=0`,
`name="a/b" isFile=false opens=0`, `name="a/b" isFile=false opens=0 size=0`,
`name="a/b/c" isFile=false opens=0`, `name="a/b/c" isFile=false opens=0 size=0`,
`name="a/b/c/d" isFile=false opens=0`, `name="a/b/c/d" isFile=false opens=0 size=0`,
`name="a/b/c/d/e" isFile=false opens=0`, `name="a/b/c/d/e" isFile=false opens=0 size=0`,
`name="a/b/c/d/e/f" isFile=false opens=0`, `name="a/b/c/d/e/f" isFile=false opens=0 size=0`,
`name="a/b/c/d/e/f/three" isFile=true opens=0`, `name="a/b/c/d/e/f/three" isFile=true opens=0 size=0`,
`name="a/b/c/d/e/two" isFile=true opens=0`, `name="a/b/c/d/e/two" isFile=true opens=0 size=0`,
`name="a/b/c/d/one" isFile=true opens=0`, `name="a/b/c/d/one" isFile=true opens=0 size=0`,
`name="potato" isFile=true opens=0`, `name="potato" isFile=true opens=0 size=0`,
}, itemAsString(c)) }, itemAsString(c))
} }
@ -289,9 +289,9 @@ func TestCacheOpenMkdir(t *testing.T) {
c.open("sub/potato") c.open("sub/potato")
assert.Equal(t, []string{ assert.Equal(t, []string{
`name="" isFile=false opens=1`, `name="" isFile=false opens=1 size=0`,
`name="sub" isFile=false opens=1`, `name="sub" isFile=false opens=1 size=0`,
`name="sub/potato" isFile=true opens=1`, `name="sub/potato" isFile=true opens=1 size=0`,
}, itemAsString(c)) }, itemAsString(c))
// mkdir // mkdir
@ -299,9 +299,9 @@ func TestCacheOpenMkdir(t *testing.T) {
require.NoError(t, err) require.NoError(t, err)
assert.Equal(t, "potato", filepath.Base(p)) assert.Equal(t, "potato", filepath.Base(p))
assert.Equal(t, []string{ assert.Equal(t, []string{
`name="" isFile=false opens=1`, `name="" isFile=false opens=1 size=0`,
`name="sub" isFile=false opens=1`, `name="sub" isFile=false opens=1 size=0`,
`name="sub/potato" isFile=true opens=1`, `name="sub/potato" isFile=true opens=1 size=0`,
}, itemAsString(c)) }, itemAsString(c))
// test directory exists // test directory exists
@ -321,13 +321,14 @@ func TestCacheOpenMkdir(t *testing.T) {
c.close("sub/potato") c.close("sub/potato")
assert.Equal(t, []string{ assert.Equal(t, []string{
`name="" isFile=false opens=0`, `name="" isFile=false opens=0 size=0`,
`name="sub" isFile=false opens=0`, `name="sub" isFile=false opens=0 size=0`,
`name="sub/potato" isFile=true opens=0`, `name="sub/potato" isFile=true opens=0 size=0`,
}, itemAsString(c)) }, itemAsString(c))
// clean the cache // clean the cache
c.purgeOld(-10 * time.Second) c.purgeOld(-10 * time.Second)
c.purgeEmptyDirs()
assert.Equal(t, []string(nil), itemAsString(c)) assert.Equal(t, []string(nil), itemAsString(c))
@ -350,24 +351,24 @@ func TestCacheCacheDir(t *testing.T) {
c.cacheDir("dir") c.cacheDir("dir")
assert.Equal(t, []string{ assert.Equal(t, []string{
`name="" isFile=false opens=0`, `name="" isFile=false opens=0 size=0`,
`name="dir" isFile=false opens=0`, `name="dir" isFile=false opens=0 size=0`,
}, itemAsString(c)) }, itemAsString(c))
c.cacheDir("dir/sub") c.cacheDir("dir/sub")
assert.Equal(t, []string{ assert.Equal(t, []string{
`name="" isFile=false opens=0`, `name="" isFile=false opens=0 size=0`,
`name="dir" isFile=false opens=0`, `name="dir" isFile=false opens=0 size=0`,
`name="dir/sub" isFile=false opens=0`, `name="dir/sub" isFile=false opens=0 size=0`,
}, itemAsString(c)) }, itemAsString(c))
c.cacheDir("dir/sub2/subsub2") c.cacheDir("dir/sub2/subsub2")
assert.Equal(t, []string{ assert.Equal(t, []string{
`name="" isFile=false opens=0`, `name="" isFile=false opens=0 size=0`,
`name="dir" isFile=false opens=0`, `name="dir" isFile=false opens=0 size=0`,
`name="dir/sub" isFile=false opens=0`, `name="dir/sub" isFile=false opens=0 size=0`,
`name="dir/sub2" isFile=false opens=0`, `name="dir/sub2" isFile=false opens=0 size=0`,
`name="dir/sub2/subsub2" isFile=false opens=0`, `name="dir/sub2/subsub2" isFile=false opens=0 size=0`,
}, itemAsString(c)) }, itemAsString(c))
} }
@ -395,7 +396,8 @@ func TestCachePurgeOld(t *testing.T) {
} }
removed = nil removed = nil
c._purgeOld(-10*time.Second, removeFile, removeDir) c._purgeOld(-10*time.Second, removeFile)
c._purgeEmptyDirs(removeDir)
assert.Equal(t, []string(nil), removed) assert.Equal(t, []string(nil), removed)
c.open("sub/dir2/potato2") c.open("sub/dir2/potato2")
@ -404,17 +406,18 @@ func TestCachePurgeOld(t *testing.T) {
c.open("sub/dir/potato") c.open("sub/dir/potato")
assert.Equal(t, []string{ assert.Equal(t, []string{
`name="" isFile=false opens=2`, `name="" isFile=false opens=2 size=0`,
`name="sub" isFile=false opens=2`, `name="sub" isFile=false opens=2 size=0`,
`name="sub/dir" isFile=false opens=2`, `name="sub/dir" isFile=false opens=2 size=0`,
`name="sub/dir/potato" isFile=true opens=2`, `name="sub/dir/potato" isFile=true opens=2 size=0`,
`name="sub/dir2" isFile=false opens=0`, `name="sub/dir2" isFile=false opens=0 size=0`,
`name="sub/dir2/potato2" isFile=true opens=0`, `name="sub/dir2/potato2" isFile=true opens=0 size=0`,
}, itemAsString(c)) }, itemAsString(c))
removed = nil removed = nil
removedDir = true removedDir = true
c._purgeOld(-10*time.Second, removeFile, removeDir) c._purgeOld(-10*time.Second, removeFile)
c._purgeEmptyDirs(removeDir)
assert.Equal(t, []string{ assert.Equal(t, []string{
"sub/dir2/potato2", "sub/dir2/potato2",
"sub/dir2/", "sub/dir2/",
@ -424,33 +427,36 @@ func TestCachePurgeOld(t *testing.T) {
removed = nil removed = nil
removedDir = true removedDir = true
c._purgeOld(-10*time.Second, removeFile, removeDir) c._purgeOld(-10*time.Second, removeFile)
c._purgeEmptyDirs(removeDir)
assert.Equal(t, []string(nil), removed) assert.Equal(t, []string(nil), removed)
c.close("sub/dir/potato") c.close("sub/dir/potato")
assert.Equal(t, []string{ assert.Equal(t, []string{
`name="" isFile=false opens=0`, `name="" isFile=false opens=0 size=0`,
`name="sub" isFile=false opens=0`, `name="sub" isFile=false opens=0 size=0`,
`name="sub/dir" isFile=false opens=0`, `name="sub/dir" isFile=false opens=0 size=0`,
`name="sub/dir/potato" isFile=true opens=0`, `name="sub/dir/potato" isFile=true opens=0 size=0`,
}, itemAsString(c)) }, itemAsString(c))
removed = nil removed = nil
removedDir = false removedDir = false
c._purgeOld(10*time.Second, removeFile, removeDir) c._purgeOld(10*time.Second, removeFile)
c._purgeEmptyDirs(removeDir)
assert.Equal(t, []string(nil), removed) assert.Equal(t, []string(nil), removed)
assert.Equal(t, []string{ assert.Equal(t, []string{
`name="" isFile=false opens=0`, `name="" isFile=false opens=0 size=0`,
`name="sub" isFile=false opens=0`, `name="sub" isFile=false opens=0 size=0`,
`name="sub/dir" isFile=false opens=0`, `name="sub/dir" isFile=false opens=0 size=0`,
`name="sub/dir/potato" isFile=true opens=0`, `name="sub/dir/potato" isFile=true opens=0 size=0`,
}, itemAsString(c)) }, itemAsString(c))
removed = nil removed = nil
removedDir = true removedDir = true
c._purgeOld(-10*time.Second, removeFile, removeDir) c._purgeOld(-10*time.Second, removeFile)
c._purgeEmptyDirs(removeDir)
assert.Equal(t, []string{ assert.Equal(t, []string{
"sub/dir/potato", "sub/dir/potato",
"sub/dir/", "sub/dir/",
@ -460,3 +466,157 @@ func TestCachePurgeOld(t *testing.T) {
assert.Equal(t, []string(nil), itemAsString(c)) assert.Equal(t, []string(nil), itemAsString(c))
} }
func TestCachePurgeOverQuota(t *testing.T) {
r := fstest.NewRun(t)
defer r.Finalise()
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
// Disable the cache cleaner as it interferes with these tests
opt := DefaultOpt
opt.CachePollInterval = 0
c, err := newCache(ctx, r.Fremote, &opt)
require.NoError(t, err)
// Test funcs
var removed []string
remove := func(name string) {
removed = append(removed, name)
c.remove(name)
}
removed = nil
c._purgeOverQuota(-1, remove)
assert.Equal(t, []string(nil), removed)
removed = nil
c._purgeOverQuota(0, remove)
assert.Equal(t, []string(nil), removed)
removed = nil
c._purgeOverQuota(1, remove)
assert.Equal(t, []string(nil), removed)
// Make some test files
c.open("sub/dir/potato")
p, err := c.mkdir("sub/dir/potato")
require.NoError(t, err)
err = ioutil.WriteFile(p, []byte("hello"), 0600)
require.NoError(t, err)
p, err = c.mkdir("sub/dir2/potato2")
c.open("sub/dir2/potato2")
require.NoError(t, err)
err = ioutil.WriteFile(p, []byte("hello2"), 0600)
require.NoError(t, err)
// make it definitely after
t1 := time.Now().Add(10 * time.Second)
c.updateStat("sub/dir2/potato2", t1, 0)
assert.Equal(t, []string{
`name="" isFile=false opens=2 size=0`,
`name="sub" isFile=false opens=2 size=0`,
`name="sub/dir" isFile=false opens=1 size=0`,
`name="sub/dir/potato" isFile=true opens=1 size=0`,
`name="sub/dir2" isFile=false opens=1 size=0`,
`name="sub/dir2/potato2" isFile=true opens=1 size=0`,
}, itemAsString(c))
// Check nothing removed
removed = nil
c._purgeOverQuota(1, remove)
assert.Equal(t, []string(nil), removed)
// Close the files
c.close("sub/dir/potato")
c.close("sub/dir2/potato2")
assert.Equal(t, []string{
`name="" isFile=false opens=0 size=0`,
`name="sub" isFile=false opens=0 size=0`,
`name="sub/dir" isFile=false opens=0 size=0`,
`name="sub/dir/potato" isFile=true opens=0 size=5`,
`name="sub/dir2" isFile=false opens=0 size=0`,
`name="sub/dir2/potato2" isFile=true opens=0 size=6`,
}, itemAsString(c))
// Update the stats to read the total size
err = c.updateStats()
require.NoError(t, err)
assert.Equal(t, int64(11), c.used)
// Check only potato removed to get below quota
removed = nil
c._purgeOverQuota(10, remove)
assert.Equal(t, []string{
"sub/dir/potato",
}, removed)
assert.Equal(t, int64(6), c.used)
assert.Equal(t, []string{
`name="" isFile=false opens=0 size=0`,
`name="sub" isFile=false opens=0 size=0`,
`name="sub/dir" isFile=false opens=0 size=0`,
`name="sub/dir2" isFile=false opens=0 size=0`,
`name="sub/dir2/potato2" isFile=true opens=0 size=6`,
}, itemAsString(c))
// Put potato back
c.open("sub/dir/potato")
p, err = c.mkdir("sub/dir/potato")
require.NoError(t, err)
err = ioutil.WriteFile(p, []byte("hello"), 0600)
require.NoError(t, err)
c.close("sub/dir/potato")
// make it definitely after
t2 := t1.Add(20 * time.Second)
c.updateStat("sub/dir/potato", t2, 5)
// Update the stats to read the total size
err = c.updateStats()
require.NoError(t, err)
assert.Equal(t, int64(11), c.used)
assert.Equal(t, []string{
`name="" isFile=false opens=0 size=0`,
`name="sub" isFile=false opens=0 size=0`,
`name="sub/dir" isFile=false opens=0 size=0`,
`name="sub/dir/potato" isFile=true opens=0 size=5`,
`name="sub/dir2" isFile=false opens=0 size=0`,
`name="sub/dir2/potato2" isFile=true opens=0 size=6`,
}, itemAsString(c))
// Check only potato2 removed to get below quota
removed = nil
c._purgeOverQuota(10, remove)
assert.Equal(t, []string{
"sub/dir2/potato2",
}, removed)
assert.Equal(t, int64(5), c.used)
c.purgeEmptyDirs()
assert.Equal(t, []string{
`name="" isFile=false opens=0 size=0`,
`name="sub" isFile=false opens=0 size=0`,
`name="sub/dir" isFile=false opens=0 size=0`,
`name="sub/dir/potato" isFile=true opens=0 size=5`,
}, itemAsString(c))
// Now purge everything
removed = nil
c._purgeOverQuota(1, remove)
assert.Equal(t, []string{
"sub/dir/potato",
}, removed)
assert.Equal(t, int64(0), c.used)
c.purgeEmptyDirs()
assert.Equal(t, []string(nil), itemAsString(c))
// Check nothing left behind
c.clean()
assert.Equal(t, int64(0), c.used)
assert.Equal(t, []string(nil), itemAsString(c))
}

6
vfs/help.go
View file

@ -60,6 +60,7 @@ may find that you need one or the other or both.
--vfs-cache-max-age duration Max age of objects in the cache. (default 1h0m0s) --vfs-cache-max-age duration Max age of objects in the cache. (default 1h0m0s)
--vfs-cache-mode string Cache mode off|minimal|writes|full (default "off") --vfs-cache-mode string Cache mode off|minimal|writes|full (default "off")
--vfs-cache-poll-interval duration Interval to poll the cache for stale objects. (default 1m0s) --vfs-cache-poll-interval duration Interval to poll the cache for stale objects. (default 1m0s)
--vfs-cache-max-size int Max total size of objects in the cache. (default off)
If run with ` + "`-vv`" + ` rclone will print the location of the file cache. The If run with ` + "`-vv`" + ` rclone will print the location of the file cache. The
files are stored in the user cache file area which is OS dependent but files are stored in the user cache file area which is OS dependent but
@ -75,6 +76,11 @@ closed so if rclone is quit or dies with open files then these won't
get written back to the remote. However they will still be in the on get written back to the remote. However they will still be in the on
disk cache. disk cache.
If using --vfs-cache-max-size note that the cache may exceed this size
for two reasons. Firstly because it is only checked every
--vfs-cache-poll-interval. Secondly because open files cannot be
evicted from the cache.
#### --vfs-cache-mode off #### --vfs-cache-mode off
In this mode the cache will read directly from the remote and write In this mode the cache will read directly from the remote and write

2
vfs/vfs.go
View file

@ -50,6 +50,7 @@ var DefaultOpt = Options{
CachePollInterval: 60 * time.Second, CachePollInterval: 60 * time.Second,
ChunkSize: 128 * fs.MebiByte, ChunkSize: 128 * fs.MebiByte,
ChunkSizeLimit: -1, ChunkSizeLimit: -1,
CacheMaxSize: -1,
} }
// Node represents either a directory (*Dir) or a file (*File) // Node represents either a directory (*Dir) or a file (*File)
@ -196,6 +197,7 @@ type Options struct {
ChunkSizeLimit fs.SizeSuffix // if > ChunkSize double the chunk size after each chunk until reached ChunkSizeLimit fs.SizeSuffix // if > ChunkSize double the chunk size after each chunk until reached
CacheMode CacheMode CacheMode CacheMode
CacheMaxAge time.Duration CacheMaxAge time.Duration
CacheMaxSize fs.SizeSuffix
CachePollInterval time.Duration CachePollInterval time.Duration
} }

1
vfs/vfsflags/vfsflags.go
View file

@ -27,6 +27,7 @@ func AddFlags(flagSet *pflag.FlagSet) {
flags.FVarP(flagSet, &Opt.CacheMode, "vfs-cache-mode", "", "Cache mode off|minimal|writes|full") flags.FVarP(flagSet, &Opt.CacheMode, "vfs-cache-mode", "", "Cache mode off|minimal|writes|full")
flags.DurationVarP(flagSet, &Opt.CachePollInterval, "vfs-cache-poll-interval", "", Opt.CachePollInterval, "Interval to poll the cache for stale objects.") flags.DurationVarP(flagSet, &Opt.CachePollInterval, "vfs-cache-poll-interval", "", Opt.CachePollInterval, "Interval to poll the cache for stale objects.")
flags.DurationVarP(flagSet, &Opt.CacheMaxAge, "vfs-cache-max-age", "", Opt.CacheMaxAge, "Max age of objects in the cache.") flags.DurationVarP(flagSet, &Opt.CacheMaxAge, "vfs-cache-max-age", "", Opt.CacheMaxAge, "Max age of objects in the cache.")
flags.FVarP(flagSet, &Opt.CacheMaxSize, "vfs-cache-max-size", "", "Max total size of objects in the cache.")
flags.FVarP(flagSet, &Opt.ChunkSize, "vfs-read-chunk-size", "", "Read the source objects in chunks.") flags.FVarP(flagSet, &Opt.ChunkSize, "vfs-read-chunk-size", "", "Read the source objects in chunks.")
flags.FVarP(flagSet, &Opt.ChunkSizeLimit, "vfs-read-chunk-size-limit", "", "If greater than --vfs-read-chunk-size, double the chunk size after each chunk read, until the limit is reached. 'off' is unlimited.") flags.FVarP(flagSet, &Opt.ChunkSizeLimit, "vfs-read-chunk-size-limit", "", "If greater than --vfs-read-chunk-size, double the chunk size after each chunk read, until the limit is reached. 'off' is unlimited.")
flags.FVarP(flagSet, DirPerms, "dir-perms", "", "Directory permissions") flags.FVarP(flagSet, DirPerms, "dir-perms", "", "Directory permissions")