vfs: Add cache reset for --vfs-cache-max-size handling at cache poll interval

The vfs-cache-max-size parameter is probably confusing to many users.
The cache cleaner checks cache size periodically at the --vfs-cache-poll-interval
(default 60 seconds) interval and remove cache items in the following order.

(1) cache items that are not in use and with age > vfs-cache-max-age
(2) if the cache space used at this time still is larger than
vfs-cache-max-size, the cleaner continues to remove cache items that are
not in use.

The cache cleaning process does not remove cache items that are currently in use.
If the total space consumed by in-use cache items exceeds vfs-cache-max-size, the
periodical cache cleaner thread does not do anything further and leaves the in-use
cache items alone with a total space larger than vfs-cache-max-size.

A cache reset feature was introduced in 1.53 which resets in-use (but not dirty,
i.e., not being updated) cache items when additional cache data incurs an ENOSPC
error.  But this code was not activated in the periodical cache cleaning thread.

This patch adds the cache reset step in the cache cleaner thread during cache
poll to reset cache items until the total size of the remaining cache items is
below vfs-cache-max-size.

go.sum

@@ -600,6 +600,7 @@ github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIK
 github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
 github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM=
 github.com/sony/gobreaker v0.4.1/go.mod h1:ZKptC7FHNvhBz7dN2LGjPVBz2sZJmc0/PkyDJOjmxWY=
+github.com/spacemonkeygo/monkit v0.0.0-20190623001553-09813957f0a8 h1:sO833wsQOMWK0ZW68mWYVhRwlcUaL125bUh1LeR7vxI=
 github.com/spacemonkeygo/monkit/v3 v3.0.4/go.mod h1:JcK1pCbReQsOsMKF/POFSZCq7drXFybgGmbc27tuwes=
 github.com/spacemonkeygo/monkit/v3 v3.0.7/go.mod h1:kj1ViJhlyADa7DiA4xVnTuPA46lFKbM7mxQTrXCuJP4=
 github.com/spacemonkeygo/monkit/v3 v3.0.10/go.mod h1:kj1ViJhlyADa7DiA4xVnTuPA46lFKbM7mxQTrXCuJP4=

vfs/vfscache/cache.go

@@ -658,7 +658,7 @@ func (c *Cache) purgeOverQuota(quota int64) {
 }
 
 // clean empties the cache of stuff if it can
-func (c *Cache) clean(removeCleanFiles bool) {
+func (c *Cache) clean(kicked bool) {
 	// Cache may be empty so end
 	_, err := os.Stat(c.root)
 	if os.IsNotExist(err) {
@@ -674,18 +674,17 @@ func (c *Cache) clean(removeCleanFiles bool) {
 		// Remove any files that are over age
 		c.purgeOld(c.opt.CacheMaxAge)
 
+		if int64(c.opt.CacheMaxSize) <= 0 {
+			break
+		}
+
 		// Now remove files not in use until cache size is below quota starting from the
 		// oldest first
 		c.purgeOverQuota(int64(c.opt.CacheMaxSize))
 
-		// removeCleanFiles indicates that we got ENOSPC error
-		// We remove cache files that are not dirty if we are still above the max cache size
-		if removeCleanFiles {
-			c.purgeClean(int64(c.opt.CacheMaxSize))
-			c.retryFailedResets()
-		} else {
-			break
-		}
+		// Remove cache files that are not dirty if we are still above the max cache size
+		c.purgeClean(int64(c.opt.CacheMaxSize))
+		c.retryFailedResets()
 
 		used := c.updateUsed()
 		if used <= int64(c.opt.CacheMaxSize) && len(c.errItems) == 0 {
@@ -694,7 +693,7 @@ func (c *Cache) clean(removeCleanFiles bool) {
 	}
 
 	// Was kicked?
-	if removeCleanFiles {
+	if kicked {
 		c.kickerMu.Lock() // Make sure this is called with cache mutex unlocked
 		// Reenable io threads to kick me
 		c.cleanerKicked = false
@@ -737,9 +736,9 @@ func (c *Cache) cleaner(ctx context.Context) {
 	for {
 		select {
 		case <-c.kick: // a thread encountering ENOSPC kicked me
-			c.clean(true) // remove inUse files that are clean (!item.info.Dirty)
+			c.clean(true) // kicked is true
 		case <-timer.C:
-			c.clean(false) // do not remove inUse files
+			c.clean(false) // timer driven cache poll, kicked is false
 		case <-ctx.Done():
 			fs.Debugf(nil, "vfs cache: cleaner exiting")
 			return