diff --git a/vfs/cache.go b/vfs/cache.go index 30b4f9289..66e994c35 100644 --- a/vfs/cache.go +++ b/vfs/cache.go @@ -67,8 +67,9 @@ type cache struct { f fs.Fs // fs for the cache directory opt *Options // vfs Options root string // root of the cache directory - itemMu sync.Mutex // protects the next two maps + itemMu sync.Mutex // protects the following variables item map[string]*cacheItem // files/directories in the cache + used int64 // total size of files in the cache } // cacheItem is stored in the item map @@ -76,6 +77,7 @@ type cacheItem struct { opens int // number of times file is open atime time.Time // last time file was accessed isFile bool // if this is a file or a directory + size int64 // size of the cached item } // newCacheItem returns an item for the cache @@ -196,11 +198,13 @@ func (c *cache) get(name string) *cacheItem { return item } -// updateTime sets the atime of the name to that passed in if it is +// updateStat sets the atime of the name to that passed in if it is // newer than the existing or there isn't an existing time. // +// it also sets the size +// // name should be a remote path not an osPath -func (c *cache) updateTime(name string, when time.Time) { +func (c *cache) updateStat(name string, when time.Time, size int64) { name = clean(name) c.itemMu.Lock() item, found := c._get(true, name) @@ -208,6 +212,7 @@ func (c *cache) updateTime(name string, when time.Time) { fs.Debugf(name, "updateTime: setting atime to %v", when) item.atime = when } + item.size = size c.itemMu.Unlock() } @@ -266,6 +271,12 @@ func (c *cache) _close(isFile bool, name string) { if item.opens < 0 { fs.Errorf(name, "cache: double close") } + osPath := c.toOSPath(name) + fi, err := os.Stat(osPath) + // Update the size on close + if err == nil && !fi.IsDir() { + item.size = fi.Size() + } if name == "" { break } @@ -291,7 +302,7 @@ func (c *cache) remove(name string) { if err != nil && !os.IsNotExist(err) { fs.Errorf(name, "Failed to remove from cache: %v", err) } else { - fs.Debugf(name, "Removed from cache") + fs.Infof(name, "Removed from cache") } } @@ -338,26 +349,34 @@ func (c *cache) walk(fn func(osPath string, fi os.FileInfo, name string) error) }) } -// updateAtimes walks the cache updating any atimes it finds -func (c *cache) updateAtimes() error { - return c.walk(func(osPath string, fi os.FileInfo, name string) error { +// updateStats walks the cache updating any atimes and sizes it finds +// +// it also updates used +func (c *cache) updateStats() error { + var newUsed int64 + err := c.walk(func(osPath string, fi os.FileInfo, name string) error { if !fi.IsDir() { // Update the atime with that of the file atime := times.Get(fi).AccessTime() - c.updateTime(name, atime) + c.updateStat(name, atime, fi.Size()) + newUsed += fi.Size() } else { c.cacheDir(name) } return nil }) + c.itemMu.Lock() + c.used = newUsed + c.itemMu.Unlock() + return err } // purgeOld gets rid of any files that are over age func (c *cache) purgeOld(maxAge time.Duration) { - c._purgeOld(maxAge, c.remove, c.removeDir) + c._purgeOld(maxAge, c.remove) } -func (c *cache) _purgeOld(maxAge time.Duration, remove func(name string), removeDir func(name string) bool) { +func (c *cache) _purgeOld(maxAge time.Duration, remove func(name string)) { c.itemMu.Lock() defer c.itemMu.Unlock() cutoff := time.Now().Add(-maxAge) @@ -373,7 +392,16 @@ func (c *cache) _purgeOld(maxAge time.Duration, remove func(name string), remove } } } - // now find any empty directories +} + +// Purge any empty directories +func (c *cache) purgeEmptyDirs() { + c._purgeEmptyDirs(c.removeDir) +} + +func (c *cache) _purgeEmptyDirs(removeDir func(name string) bool) { + c.itemMu.Lock() + defer c.itemMu.Unlock() var dirs []string for name, item := range c.item { if !item.isFile && item.opens == 0 { @@ -391,6 +419,56 @@ func (c *cache) _purgeOld(maxAge time.Duration, remove func(name string), remove } } +// This is a cacheItem with a name for sorting +type cacheNamedItem struct { + name string + item *cacheItem +} +type cacheNamedItems []cacheNamedItem + +func (v cacheNamedItems) Len() int { return len(v) } +func (v cacheNamedItems) Swap(i, j int) { v[i], v[j] = v[j], v[i] } +func (v cacheNamedItems) Less(i, j int) bool { return v[i].item.atime.Before(v[j].item.atime) } + +// Remove any files that are over quota starting from the +// oldest first +func (c *cache) purgeOverQuota(quota int64) { + c._purgeOverQuota(quota, c.remove) +} + +func (c *cache) _purgeOverQuota(quota int64, remove func(name string)) { + c.itemMu.Lock() + defer c.itemMu.Unlock() + + if quota <= 0 || c.used < quota { + return + } + + var items cacheNamedItems + + // Make a slice of unused files + for name, item := range c.item { + if item.isFile && item.opens == 0 { + items = append(items, cacheNamedItem{ + name: name, + item: item, + }) + } + } + sort.Sort(items) + + // Remove items until the quota is OK + for _, item := range items { + if c.used < quota { + break + } + remove(item.name) + // Remove the entry + delete(c.item, item.name) + c.used -= item.item.size + } +} + // clean empties the cache of stuff if it can func (c *cache) clean() { // Cache may be empty so end @@ -399,17 +477,32 @@ func (c *cache) clean() { return } - fs.Debugf(nil, "Cleaning the cache") + c.itemMu.Lock() + oldItems, oldUsed := len(c.item), fs.SizeSuffix(c.used) + c.itemMu.Unlock() - // first walk the FS to update the atimes - err = c.updateAtimes() + // first walk the FS to update the atimes and sizes + err = c.updateStats() if err != nil { fs.Errorf(nil, "Error traversing cache %q: %v", c.root, err) } - // Now remove any files that are over age and any empty - // directories + // Remove any files that are over age c.purgeOld(c.opt.CacheMaxAge) + + // Now remove any files that are over quota starting from the + // oldest first + c.purgeOverQuota(int64(c.opt.CacheMaxSize)) + + // Remove any empty directories + c.purgeEmptyDirs() + + // Stats + c.itemMu.Lock() + newItems, newUsed := len(c.item), fs.SizeSuffix(c.used) + c.itemMu.Unlock() + + fs.Infof(nil, "Cleaned the cache: objects %d (was %d), total size %v (was %v)", newItems, oldItems, newUsed, oldUsed) } // cleaner calls clean at regular intervals diff --git a/vfs/cache_test.go b/vfs/cache_test.go index 6db0d2599..918261a0b 100644 --- a/vfs/cache_test.go +++ b/vfs/cache_test.go @@ -51,7 +51,7 @@ func itemAsString(c *cache) []string { defer c.itemMu.Unlock() var out []string for name, item := range c.item { - out = append(out, fmt.Sprintf("name=%q isFile=%v opens=%d", name, item.isFile, item.opens)) + out = append(out, fmt.Sprintf("name=%q isFile=%v opens=%d size=%d", name, item.isFile, item.opens, item.size)) } sort.Strings(out) return out @@ -79,7 +79,7 @@ func TestCacheNew(t *testing.T) { require.NoError(t, err) assert.Equal(t, "potato", filepath.Base(p)) assert.Equal(t, []string{ - `name="" isFile=false opens=0`, + `name="" isFile=false opens=0 size=0`, }, itemAsString(c)) fi, err := os.Stat(filepath.Dir(p)) @@ -95,26 +95,26 @@ func TestCacheNew(t *testing.T) { // updateTime //.. before t1 := time.Now().Add(-60 * time.Minute) - c.updateTime("potato", t1) + c.updateStat("potato", t1, 0) item = c.get("potato") assert.NotEqual(t, t1, item.atime) assert.Equal(t, 0, item.opens) //..after t2 := time.Now().Add(60 * time.Minute) - c.updateTime("potato", t2) + c.updateStat("potato", t2, 0) item = c.get("potato") assert.Equal(t, t2, item.atime) assert.Equal(t, 0, item.opens) // open assert.Equal(t, []string{ - `name="" isFile=false opens=0`, - `name="potato" isFile=true opens=0`, + `name="" isFile=false opens=0 size=0`, + `name="potato" isFile=true opens=0 size=0`, }, itemAsString(c)) c.open("/potato") assert.Equal(t, []string{ - `name="" isFile=false opens=1`, - `name="potato" isFile=true opens=1`, + `name="" isFile=false opens=1 size=0`, + `name="potato" isFile=true opens=1 size=0`, }, itemAsString(c)) item = c.get("potato") assert.WithinDuration(t, time.Now(), item.atime, time.Second) @@ -132,11 +132,11 @@ func TestCacheNew(t *testing.T) { // updateAtimes item = c.get("potato") item.atime = time.Now().Add(-24 * time.Hour) - err = c.updateAtimes() + err = c.updateStats() require.NoError(t, err) assert.Equal(t, []string{ - `name="" isFile=false opens=1`, - `name="potato" isFile=true opens=1`, + `name="" isFile=false opens=1 size=0`, + `name="potato" isFile=true opens=1 size=5`, }, itemAsString(c)) item = c.get("potato") assert.Equal(t, atime, item.atime) @@ -146,11 +146,11 @@ func TestCacheNew(t *testing.T) { c.itemMu.Lock() delete(c.item, "potato") // remove from cache c.itemMu.Unlock() - err = c.updateAtimes() + err = c.updateStats() require.NoError(t, err) assert.Equal(t, []string{ - `name="" isFile=false opens=1`, - `name="potato" isFile=true opens=0`, + `name="" isFile=false opens=1 size=0`, + `name="potato" isFile=true opens=0 size=5`, }, itemAsString(c)) item = c.get("potato") assert.Equal(t, atime, item.atime) @@ -165,18 +165,18 @@ func TestCacheNew(t *testing.T) { // close assert.Equal(t, []string{ - `name="" isFile=false opens=1`, - `name="potato" isFile=true opens=1`, + `name="" isFile=false opens=1 size=0`, + `name="potato" isFile=true opens=1 size=5`, }, itemAsString(c)) - c.updateTime("potato", t2) + c.updateStat("potato", t2, 6) assert.Equal(t, []string{ - `name="" isFile=false opens=1`, - `name="potato" isFile=true opens=1`, + `name="" isFile=false opens=1 size=0`, + `name="potato" isFile=true opens=1 size=6`, }, itemAsString(c)) c.close("potato/") assert.Equal(t, []string{ - `name="" isFile=false opens=0`, - `name="potato" isFile=true opens=0`, + `name="" isFile=false opens=0 size=0`, + `name="potato" isFile=true opens=0 size=5`, }, itemAsString(c)) item = c.get("potato") assert.WithinDuration(t, time.Now(), item.atime, time.Second) @@ -216,23 +216,23 @@ func TestCacheOpens(t *testing.T) { assert.Equal(t, []string(nil), itemAsString(c)) c.open("potato") assert.Equal(t, []string{ - `name="" isFile=false opens=1`, - `name="potato" isFile=true opens=1`, + `name="" isFile=false opens=1 size=0`, + `name="potato" isFile=true opens=1 size=0`, }, itemAsString(c)) c.open("potato") assert.Equal(t, []string{ - `name="" isFile=false opens=2`, - `name="potato" isFile=true opens=2`, + `name="" isFile=false opens=2 size=0`, + `name="potato" isFile=true opens=2 size=0`, }, itemAsString(c)) c.close("potato") assert.Equal(t, []string{ - `name="" isFile=false opens=1`, - `name="potato" isFile=true opens=1`, + `name="" isFile=false opens=1 size=0`, + `name="potato" isFile=true opens=1 size=0`, }, itemAsString(c)) c.close("potato") assert.Equal(t, []string{ - `name="" isFile=false opens=0`, - `name="potato" isFile=true opens=0`, + `name="" isFile=false opens=0 size=0`, + `name="potato" isFile=true opens=0 size=0`, }, itemAsString(c)) c.open("potato") @@ -240,34 +240,34 @@ func TestCacheOpens(t *testing.T) { c.open("a/b/c/d/e/two") c.open("a/b/c/d/e/f/three") assert.Equal(t, []string{ - `name="" isFile=false opens=4`, - `name="a" isFile=false opens=3`, - `name="a/b" isFile=false opens=3`, - `name="a/b/c" isFile=false opens=3`, - `name="a/b/c/d" isFile=false opens=3`, - `name="a/b/c/d/e" isFile=false opens=2`, - `name="a/b/c/d/e/f" isFile=false opens=1`, - `name="a/b/c/d/e/f/three" isFile=true opens=1`, - `name="a/b/c/d/e/two" isFile=true opens=1`, - `name="a/b/c/d/one" isFile=true opens=1`, - `name="potato" isFile=true opens=1`, + `name="" isFile=false opens=4 size=0`, + `name="a" isFile=false opens=3 size=0`, + `name="a/b" isFile=false opens=3 size=0`, + `name="a/b/c" isFile=false opens=3 size=0`, + `name="a/b/c/d" isFile=false opens=3 size=0`, + `name="a/b/c/d/e" isFile=false opens=2 size=0`, + `name="a/b/c/d/e/f" isFile=false opens=1 size=0`, + `name="a/b/c/d/e/f/three" isFile=true opens=1 size=0`, + `name="a/b/c/d/e/two" isFile=true opens=1 size=0`, + `name="a/b/c/d/one" isFile=true opens=1 size=0`, + `name="potato" isFile=true opens=1 size=0`, }, itemAsString(c)) c.close("potato") c.close("a/b/c/d/one") c.close("a/b/c/d/e/two") c.close("a/b/c//d/e/f/three") assert.Equal(t, []string{ - `name="" isFile=false opens=0`, - `name="a" isFile=false opens=0`, - `name="a/b" isFile=false opens=0`, - `name="a/b/c" isFile=false opens=0`, - `name="a/b/c/d" isFile=false opens=0`, - `name="a/b/c/d/e" isFile=false opens=0`, - `name="a/b/c/d/e/f" isFile=false opens=0`, - `name="a/b/c/d/e/f/three" isFile=true opens=0`, - `name="a/b/c/d/e/two" isFile=true opens=0`, - `name="a/b/c/d/one" isFile=true opens=0`, - `name="potato" isFile=true opens=0`, + `name="" isFile=false opens=0 size=0`, + `name="a" isFile=false opens=0 size=0`, + `name="a/b" isFile=false opens=0 size=0`, + `name="a/b/c" isFile=false opens=0 size=0`, + `name="a/b/c/d" isFile=false opens=0 size=0`, + `name="a/b/c/d/e" isFile=false opens=0 size=0`, + `name="a/b/c/d/e/f" isFile=false opens=0 size=0`, + `name="a/b/c/d/e/f/three" isFile=true opens=0 size=0`, + `name="a/b/c/d/e/two" isFile=true opens=0 size=0`, + `name="a/b/c/d/one" isFile=true opens=0 size=0`, + `name="potato" isFile=true opens=0 size=0`, }, itemAsString(c)) } @@ -289,9 +289,9 @@ func TestCacheOpenMkdir(t *testing.T) { c.open("sub/potato") assert.Equal(t, []string{ - `name="" isFile=false opens=1`, - `name="sub" isFile=false opens=1`, - `name="sub/potato" isFile=true opens=1`, + `name="" isFile=false opens=1 size=0`, + `name="sub" isFile=false opens=1 size=0`, + `name="sub/potato" isFile=true opens=1 size=0`, }, itemAsString(c)) // mkdir @@ -299,9 +299,9 @@ func TestCacheOpenMkdir(t *testing.T) { require.NoError(t, err) assert.Equal(t, "potato", filepath.Base(p)) assert.Equal(t, []string{ - `name="" isFile=false opens=1`, - `name="sub" isFile=false opens=1`, - `name="sub/potato" isFile=true opens=1`, + `name="" isFile=false opens=1 size=0`, + `name="sub" isFile=false opens=1 size=0`, + `name="sub/potato" isFile=true opens=1 size=0`, }, itemAsString(c)) // test directory exists @@ -321,13 +321,14 @@ func TestCacheOpenMkdir(t *testing.T) { c.close("sub/potato") assert.Equal(t, []string{ - `name="" isFile=false opens=0`, - `name="sub" isFile=false opens=0`, - `name="sub/potato" isFile=true opens=0`, + `name="" isFile=false opens=0 size=0`, + `name="sub" isFile=false opens=0 size=0`, + `name="sub/potato" isFile=true opens=0 size=0`, }, itemAsString(c)) // clean the cache c.purgeOld(-10 * time.Second) + c.purgeEmptyDirs() assert.Equal(t, []string(nil), itemAsString(c)) @@ -350,24 +351,24 @@ func TestCacheCacheDir(t *testing.T) { c.cacheDir("dir") assert.Equal(t, []string{ - `name="" isFile=false opens=0`, - `name="dir" isFile=false opens=0`, + `name="" isFile=false opens=0 size=0`, + `name="dir" isFile=false opens=0 size=0`, }, itemAsString(c)) c.cacheDir("dir/sub") assert.Equal(t, []string{ - `name="" isFile=false opens=0`, - `name="dir" isFile=false opens=0`, - `name="dir/sub" isFile=false opens=0`, + `name="" isFile=false opens=0 size=0`, + `name="dir" isFile=false opens=0 size=0`, + `name="dir/sub" isFile=false opens=0 size=0`, }, itemAsString(c)) c.cacheDir("dir/sub2/subsub2") assert.Equal(t, []string{ - `name="" isFile=false opens=0`, - `name="dir" isFile=false opens=0`, - `name="dir/sub" isFile=false opens=0`, - `name="dir/sub2" isFile=false opens=0`, - `name="dir/sub2/subsub2" isFile=false opens=0`, + `name="" isFile=false opens=0 size=0`, + `name="dir" isFile=false opens=0 size=0`, + `name="dir/sub" isFile=false opens=0 size=0`, + `name="dir/sub2" isFile=false opens=0 size=0`, + `name="dir/sub2/subsub2" isFile=false opens=0 size=0`, }, itemAsString(c)) } @@ -395,7 +396,8 @@ func TestCachePurgeOld(t *testing.T) { } removed = nil - c._purgeOld(-10*time.Second, removeFile, removeDir) + c._purgeOld(-10*time.Second, removeFile) + c._purgeEmptyDirs(removeDir) assert.Equal(t, []string(nil), removed) c.open("sub/dir2/potato2") @@ -404,17 +406,18 @@ func TestCachePurgeOld(t *testing.T) { c.open("sub/dir/potato") assert.Equal(t, []string{ - `name="" isFile=false opens=2`, - `name="sub" isFile=false opens=2`, - `name="sub/dir" isFile=false opens=2`, - `name="sub/dir/potato" isFile=true opens=2`, - `name="sub/dir2" isFile=false opens=0`, - `name="sub/dir2/potato2" isFile=true opens=0`, + `name="" isFile=false opens=2 size=0`, + `name="sub" isFile=false opens=2 size=0`, + `name="sub/dir" isFile=false opens=2 size=0`, + `name="sub/dir/potato" isFile=true opens=2 size=0`, + `name="sub/dir2" isFile=false opens=0 size=0`, + `name="sub/dir2/potato2" isFile=true opens=0 size=0`, }, itemAsString(c)) removed = nil removedDir = true - c._purgeOld(-10*time.Second, removeFile, removeDir) + c._purgeOld(-10*time.Second, removeFile) + c._purgeEmptyDirs(removeDir) assert.Equal(t, []string{ "sub/dir2/potato2", "sub/dir2/", @@ -424,33 +427,36 @@ func TestCachePurgeOld(t *testing.T) { removed = nil removedDir = true - c._purgeOld(-10*time.Second, removeFile, removeDir) + c._purgeOld(-10*time.Second, removeFile) + c._purgeEmptyDirs(removeDir) assert.Equal(t, []string(nil), removed) c.close("sub/dir/potato") assert.Equal(t, []string{ - `name="" isFile=false opens=0`, - `name="sub" isFile=false opens=0`, - `name="sub/dir" isFile=false opens=0`, - `name="sub/dir/potato" isFile=true opens=0`, + `name="" isFile=false opens=0 size=0`, + `name="sub" isFile=false opens=0 size=0`, + `name="sub/dir" isFile=false opens=0 size=0`, + `name="sub/dir/potato" isFile=true opens=0 size=0`, }, itemAsString(c)) removed = nil removedDir = false - c._purgeOld(10*time.Second, removeFile, removeDir) + c._purgeOld(10*time.Second, removeFile) + c._purgeEmptyDirs(removeDir) assert.Equal(t, []string(nil), removed) assert.Equal(t, []string{ - `name="" isFile=false opens=0`, - `name="sub" isFile=false opens=0`, - `name="sub/dir" isFile=false opens=0`, - `name="sub/dir/potato" isFile=true opens=0`, + `name="" isFile=false opens=0 size=0`, + `name="sub" isFile=false opens=0 size=0`, + `name="sub/dir" isFile=false opens=0 size=0`, + `name="sub/dir/potato" isFile=true opens=0 size=0`, }, itemAsString(c)) removed = nil removedDir = true - c._purgeOld(-10*time.Second, removeFile, removeDir) + c._purgeOld(-10*time.Second, removeFile) + c._purgeEmptyDirs(removeDir) assert.Equal(t, []string{ "sub/dir/potato", "sub/dir/", @@ -460,3 +466,157 @@ func TestCachePurgeOld(t *testing.T) { assert.Equal(t, []string(nil), itemAsString(c)) } + +func TestCachePurgeOverQuota(t *testing.T) { + r := fstest.NewRun(t) + defer r.Finalise() + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + // Disable the cache cleaner as it interferes with these tests + opt := DefaultOpt + opt.CachePollInterval = 0 + c, err := newCache(ctx, r.Fremote, &opt) + require.NoError(t, err) + + // Test funcs + var removed []string + remove := func(name string) { + removed = append(removed, name) + c.remove(name) + } + + removed = nil + c._purgeOverQuota(-1, remove) + assert.Equal(t, []string(nil), removed) + + removed = nil + c._purgeOverQuota(0, remove) + assert.Equal(t, []string(nil), removed) + + removed = nil + c._purgeOverQuota(1, remove) + assert.Equal(t, []string(nil), removed) + + // Make some test files + c.open("sub/dir/potato") + p, err := c.mkdir("sub/dir/potato") + require.NoError(t, err) + err = ioutil.WriteFile(p, []byte("hello"), 0600) + require.NoError(t, err) + + p, err = c.mkdir("sub/dir2/potato2") + c.open("sub/dir2/potato2") + require.NoError(t, err) + err = ioutil.WriteFile(p, []byte("hello2"), 0600) + require.NoError(t, err) + // make it definitely after + t1 := time.Now().Add(10 * time.Second) + c.updateStat("sub/dir2/potato2", t1, 0) + + assert.Equal(t, []string{ + `name="" isFile=false opens=2 size=0`, + `name="sub" isFile=false opens=2 size=0`, + `name="sub/dir" isFile=false opens=1 size=0`, + `name="sub/dir/potato" isFile=true opens=1 size=0`, + `name="sub/dir2" isFile=false opens=1 size=0`, + `name="sub/dir2/potato2" isFile=true opens=1 size=0`, + }, itemAsString(c)) + + // Check nothing removed + removed = nil + c._purgeOverQuota(1, remove) + assert.Equal(t, []string(nil), removed) + + // Close the files + c.close("sub/dir/potato") + c.close("sub/dir2/potato2") + + assert.Equal(t, []string{ + `name="" isFile=false opens=0 size=0`, + `name="sub" isFile=false opens=0 size=0`, + `name="sub/dir" isFile=false opens=0 size=0`, + `name="sub/dir/potato" isFile=true opens=0 size=5`, + `name="sub/dir2" isFile=false opens=0 size=0`, + `name="sub/dir2/potato2" isFile=true opens=0 size=6`, + }, itemAsString(c)) + + // Update the stats to read the total size + err = c.updateStats() + require.NoError(t, err) + assert.Equal(t, int64(11), c.used) + + // Check only potato removed to get below quota + removed = nil + c._purgeOverQuota(10, remove) + assert.Equal(t, []string{ + "sub/dir/potato", + }, removed) + assert.Equal(t, int64(6), c.used) + + assert.Equal(t, []string{ + `name="" isFile=false opens=0 size=0`, + `name="sub" isFile=false opens=0 size=0`, + `name="sub/dir" isFile=false opens=0 size=0`, + `name="sub/dir2" isFile=false opens=0 size=0`, + `name="sub/dir2/potato2" isFile=true opens=0 size=6`, + }, itemAsString(c)) + + // Put potato back + c.open("sub/dir/potato") + p, err = c.mkdir("sub/dir/potato") + require.NoError(t, err) + err = ioutil.WriteFile(p, []byte("hello"), 0600) + require.NoError(t, err) + c.close("sub/dir/potato") + // make it definitely after + t2 := t1.Add(20 * time.Second) + c.updateStat("sub/dir/potato", t2, 5) + + // Update the stats to read the total size + err = c.updateStats() + require.NoError(t, err) + assert.Equal(t, int64(11), c.used) + + assert.Equal(t, []string{ + `name="" isFile=false opens=0 size=0`, + `name="sub" isFile=false opens=0 size=0`, + `name="sub/dir" isFile=false opens=0 size=0`, + `name="sub/dir/potato" isFile=true opens=0 size=5`, + `name="sub/dir2" isFile=false opens=0 size=0`, + `name="sub/dir2/potato2" isFile=true opens=0 size=6`, + }, itemAsString(c)) + + // Check only potato2 removed to get below quota + removed = nil + c._purgeOverQuota(10, remove) + assert.Equal(t, []string{ + "sub/dir2/potato2", + }, removed) + assert.Equal(t, int64(5), c.used) + c.purgeEmptyDirs() + + assert.Equal(t, []string{ + `name="" isFile=false opens=0 size=0`, + `name="sub" isFile=false opens=0 size=0`, + `name="sub/dir" isFile=false opens=0 size=0`, + `name="sub/dir/potato" isFile=true opens=0 size=5`, + }, itemAsString(c)) + + // Now purge everything + removed = nil + c._purgeOverQuota(1, remove) + assert.Equal(t, []string{ + "sub/dir/potato", + }, removed) + assert.Equal(t, int64(0), c.used) + c.purgeEmptyDirs() + + assert.Equal(t, []string(nil), itemAsString(c)) + + // Check nothing left behind + c.clean() + assert.Equal(t, int64(0), c.used) + assert.Equal(t, []string(nil), itemAsString(c)) +} diff --git a/vfs/help.go b/vfs/help.go index c9c423655..180965aa3 100644 --- a/vfs/help.go +++ b/vfs/help.go @@ -60,6 +60,7 @@ may find that you need one or the other or both. --vfs-cache-max-age duration Max age of objects in the cache. (default 1h0m0s) --vfs-cache-mode string Cache mode off|minimal|writes|full (default "off") --vfs-cache-poll-interval duration Interval to poll the cache for stale objects. (default 1m0s) + --vfs-cache-max-size int Max total size of objects in the cache. (default off) If run with ` + "`-vv`" + ` rclone will print the location of the file cache. The files are stored in the user cache file area which is OS dependent but @@ -75,6 +76,11 @@ closed so if rclone is quit or dies with open files then these won't get written back to the remote. However they will still be in the on disk cache. +If using --vfs-cache-max-size note that the cache may exceed this size +for two reasons. Firstly because it is only checked every +--vfs-cache-poll-interval. Secondly because open files cannot be +evicted from the cache. + #### --vfs-cache-mode off In this mode the cache will read directly from the remote and write diff --git a/vfs/vfs.go b/vfs/vfs.go index ea39e0753..31f9d56c9 100644 --- a/vfs/vfs.go +++ b/vfs/vfs.go @@ -50,6 +50,7 @@ var DefaultOpt = Options{ CachePollInterval: 60 * time.Second, ChunkSize: 128 * fs.MebiByte, ChunkSizeLimit: -1, + CacheMaxSize: -1, } // Node represents either a directory (*Dir) or a file (*File) @@ -196,6 +197,7 @@ type Options struct { ChunkSizeLimit fs.SizeSuffix // if > ChunkSize double the chunk size after each chunk until reached CacheMode CacheMode CacheMaxAge time.Duration + CacheMaxSize fs.SizeSuffix CachePollInterval time.Duration } diff --git a/vfs/vfsflags/vfsflags.go b/vfs/vfsflags/vfsflags.go index b216b16c7..a8244a49a 100644 --- a/vfs/vfsflags/vfsflags.go +++ b/vfs/vfsflags/vfsflags.go @@ -27,6 +27,7 @@ func AddFlags(flagSet *pflag.FlagSet) { flags.FVarP(flagSet, &Opt.CacheMode, "vfs-cache-mode", "", "Cache mode off|minimal|writes|full") flags.DurationVarP(flagSet, &Opt.CachePollInterval, "vfs-cache-poll-interval", "", Opt.CachePollInterval, "Interval to poll the cache for stale objects.") flags.DurationVarP(flagSet, &Opt.CacheMaxAge, "vfs-cache-max-age", "", Opt.CacheMaxAge, "Max age of objects in the cache.") + flags.FVarP(flagSet, &Opt.CacheMaxSize, "vfs-cache-max-size", "", "Max total size of objects in the cache.") flags.FVarP(flagSet, &Opt.ChunkSize, "vfs-read-chunk-size", "", "Read the source objects in chunks.") flags.FVarP(flagSet, &Opt.ChunkSizeLimit, "vfs-read-chunk-size-limit", "", "If greater than --vfs-read-chunk-size, double the chunk size after each chunk read, until the limit is reached. 'off' is unlimited.") flags.FVarP(flagSet, DirPerms, "dir-perms", "", "Directory permissions")