From 712f7e38f73d237e2554aa98cd5d0bfdb396d4c9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Micha=C5=82=20Matczuk?= <michal@scylladb.com>
Date: Wed, 14 Aug 2019 10:40:43 +0200
Subject: [PATCH] backend/local: fadvise run syscall on a dedicated go routine

Before we issued an additional syscall periodically on a hot path.
This patch offloads the fadvise syscall to a dedicated go routine.
---
 backend/local/fadvise_unix.go | 62 +++++++++++++++++++++++++++--------
 1 file changed, 49 insertions(+), 13 deletions(-)

diff --git a/backend/local/fadvise_unix.go b/backend/local/fadvise_unix.go
index 7a6f4bad5..fce828971 100644
--- a/backend/local/fadvise_unix.go
+++ b/backend/local/fadvise_unix.go
@@ -45,6 +45,35 @@ type fadvise struct {
 	lastPos    int64
 	curPos     int64
 	windowSize int64
+
+	freePagesCh chan offsetLength
+	doneCh      chan struct{}
+}
+
+type offsetLength struct {
+	offset int64
+	length int64
+}
+
+const (
+	defaultAllowPages      = 32
+	defaultWorkerQueueSize = 64
+)
+
+func newFadvise(o *Object, fd int, offset int64) *fadvise {
+	f := &fadvise{
+		o:          o,
+		fd:         fd,
+		lastPos:    offset,
+		curPos:     offset,
+		windowSize: int64(os.Getpagesize()) * defaultAllowPages,
+
+		freePagesCh: make(chan offsetLength, defaultWorkerQueueSize),
+		doneCh:      make(chan struct{}),
+	}
+	go f.worker()
+
+	return f
 }
 
 // sequential configures readahead strategy in Linux kernel.
@@ -77,19 +106,30 @@ func (f *fadvise) freePagesIfNeeded() {
 }
 
 func (f *fadvise) freePages() {
-	if err := unix.Fadvise(f.fd, f.lastPos, f.curPos-f.lastPos, unix.FADV_DONTNEED); err != nil {
-		fs.Debugf(f.o, "fadvise dontneed failed on file descriptor %d: %s", f.fd, err)
-	}
+	f.freePagesCh <- offsetLength{f.lastPos, f.curPos - f.lastPos}
 	f.lastPos = f.curPos
 }
 
+func (f *fadvise) worker() {
+	for p := range f.freePagesCh {
+		if err := unix.Fadvise(f.fd, p.offset, p.length, unix.FADV_DONTNEED); err != nil {
+			fs.Debugf(f.o, "fadvise dontneed failed on file descriptor %d: %s", f.fd, err)
+		}
+	}
+
+	close(f.doneCh)
+}
+
+func (f *fadvise) wait() {
+	close(f.freePagesCh)
+	<-f.doneCh
+}
+
 type fadviseReadCloser struct {
 	*fadvise
 	inner io.ReadCloser
 }
 
-var defaultWindowSize = int64(32 * os.Getpagesize())
-
 // newFadviseReadCloser wraps os.File so that reading from that file would
 // remove already consumed pages from kernel page cache.
 // In addition to that it instructs kernel to double the readahead window to
@@ -97,20 +137,15 @@ var defaultWindowSize = int64(32 * os.Getpagesize())
 // See also fadvise.
 func newFadviseReadCloser(o *Object, f *os.File, offset, limit int64) io.ReadCloser {
 	r := fadviseReadCloser{
-		fadvise: &fadvise{
-			o:          o,
-			fd:         int(f.Fd()),
-			lastPos:    offset,
-			curPos:     offset,
-			windowSize: defaultWindowSize,
-		},
-		inner: f,
+		fadvise: newFadvise(o, int(f.Fd()), offset),
+		inner:   f,
 	}
 
 	// If syscall failed it's likely that the subsequent syscalls to that
 	// file descriptor would also fail. In that case return the provided os.File
 	// pointer.
 	if !r.sequential(limit) {
+		r.wait()
 		return f
 	}
 
@@ -125,5 +160,6 @@ func (f fadviseReadCloser) Read(p []byte) (n int, err error) {
 
 func (f fadviseReadCloser) Close() error {
 	f.freePages()
+	f.wait()
 	return f.inner.Close()
 }