From 2295123cad41698d9e1f3b96dfc7356cdabdfef3 Mon Sep 17 00:00:00 2001
From: Leo Luan <leoluan@us.ibm.com>
Date: Mon, 5 Oct 2020 01:47:54 -0700
Subject: [PATCH] vfs: Add exponential backoff during ENOSPC retries

Add an exponentially increasing delay during retries up ENOSPC error
to avoid exhausting the 10 retries too soon when the cache space
recovery from item resets is not available from the file system yet
or consumed by other large cache writes.
---
 vfs/vfscache/cache.go | 1 -
 vfs/vfscache/item.go  | 7 +++++++
 2 files changed, 7 insertions(+), 1 deletion(-)

diff --git a/vfs/vfscache/cache.go b/vfs/vfscache/cache.go
index 9f6d5ca4e..8faf0e6a7 100644
--- a/vfs/vfscache/cache.go
+++ b/vfs/vfscache/cache.go
@@ -610,7 +610,6 @@ func (c *Cache) clean(removeCleanFiles bool) {
 	if os.IsNotExist(err) {
 		return
 	}
-
 	c.mu.Lock()
 	oldItems, oldUsed := len(c.item), fs.SizeSuffix(c.used)
 	c.mu.Unlock()
diff --git a/vfs/vfscache/item.go b/vfs/vfscache/item.go
index 2ca929b95..22ca1801d 100644
--- a/vfs/vfscache/item.go
+++ b/vfs/vfscache/item.go
@@ -1182,6 +1182,7 @@ func (item *Item) setModTime(modTime time.Time) {
 // ReadAt bytes from the file at off
 func (item *Item) ReadAt(b []byte, off int64) (n int, err error) {
 	n = 0
+	var expBackOff int
 	for retries := 0; retries < fs.Config.LowLevelRetries; retries++ {
 		item.preAccess()
 		n, err = item.readAt(b, off)
@@ -1195,6 +1196,12 @@ func (item *Item) ReadAt(b []byte, off int64) (n int, err error) {
 			break
 		}
 		item.c.KickCleaner()
+		expBackOff = 2 << uint(retries)
+		time.Sleep(time.Duration(expBackOff) * time.Millisecond) // Exponential back-off the retries
+	}
+
+	if fserrors.IsErrNoSpace(err) {
+		fs.Errorf(item.name, "vfs cache: failed to _ensure cache after retries %v", err)
 	}
 
 	return n, err