Break the fs package up into smaller parts.

The purpose of this is to make it easier to maintain and eventually to
allow the rclone backends to be re-used in other projects without
having to use the rclone configuration system.

The new code layout is documented in CONTRIBUTING.

fs/asyncreader/asyncreader.go

@@ -0,0 +1,273 @@
+// Package asyncreader provides an asynchronous reader which reads
+// independently of write
+package asyncreader
+
+import (
+	"io"
+	"sync"
+
+	"github.com/ncw/rclone/lib/readers"
+	"github.com/pkg/errors"
+)
+
+const (
+	// BufferSize is the default size of the async buffer
+	BufferSize       = 1024 * 1024
+	softStartInitial = 4 * 1024
+)
+
+var asyncBufferPool = sync.Pool{
+	New: func() interface{} { return newBuffer() },
+}
+
+var errorStreamAbandoned = errors.New("stream abandoned")
+
+// AsyncReader will do async read-ahead from the input reader
+// and make the data available as an io.Reader.
+// This should be fully transparent, except that once an error
+// has been returned from the Reader, it will not recover.
+type AsyncReader struct {
+	in      io.ReadCloser // Input reader
+	ready   chan *buffer  // Buffers ready to be handed to the reader
+	token   chan struct{} // Tokens which allow a buffer to be taken
+	exit    chan struct{} // Closes when finished
+	buffers int           // Number of buffers
+	err     error         // If an error has occurred it is here
+	cur     *buffer       // Current buffer being served
+	exited  chan struct{} // Channel is closed been the async reader shuts down
+	size    int           // size of buffer to use
+	closed  bool          // whether we have closed the underlying stream
+	mu      sync.Mutex    // lock for Read/WriteTo/Abandon/Close
+}
+
+// New returns a reader that will asynchronously read from
+// the supplied Reader into a number of buffers each of size BufferSize
+// It will start reading from the input at once, maybe even before this
+// function has returned.
+// The input can be read from the returned reader.
+// When done use Close to release the buffers and close the supplied input.
+func New(rd io.ReadCloser, buffers int) (*AsyncReader, error) {
+	if buffers <= 0 {
+		return nil, errors.New("number of buffers too small")
+	}
+	if rd == nil {
+		return nil, errors.New("nil reader supplied")
+	}
+	a := &AsyncReader{}
+	a.init(rd, buffers)
+	return a, nil
+}
+
+func (a *AsyncReader) init(rd io.ReadCloser, buffers int) {
+	a.in = rd
+	a.ready = make(chan *buffer, buffers)
+	a.token = make(chan struct{}, buffers)
+	a.exit = make(chan struct{}, 0)
+	a.exited = make(chan struct{}, 0)
+	a.buffers = buffers
+	a.cur = nil
+	a.size = softStartInitial
+
+	// Create tokens
+	for i := 0; i < buffers; i++ {
+		a.token <- struct{}{}
+	}
+
+	// Start async reader
+	go func() {
+		// Ensure that when we exit this is signalled.
+		defer close(a.exited)
+		defer close(a.ready)
+		for {
+			select {
+			case <-a.token:
+				b := a.getBuffer()
+				if a.size < BufferSize {
+					b.buf = b.buf[:a.size]
+					a.size <<= 1
+				}
+				err := b.read(a.in)
+				a.ready <- b
+				if err != nil {
+					return
+				}
+			case <-a.exit:
+				return
+			}
+		}
+	}()
+}
+
+// return the buffer to the pool (clearing it)
+func (a *AsyncReader) putBuffer(b *buffer) {
+	b.clear()
+	asyncBufferPool.Put(b)
+}
+
+// get a buffer from the pool
+func (a *AsyncReader) getBuffer() *buffer {
+	b := asyncBufferPool.Get().(*buffer)
+	return b
+}
+
+// Read will return the next available data.
+func (a *AsyncReader) fill() (err error) {
+	if a.cur.isEmpty() {
+		if a.cur != nil {
+			a.putBuffer(a.cur)
+			a.token <- struct{}{}
+			a.cur = nil
+		}
+		b, ok := <-a.ready
+		if !ok {
+			// Return an error to show fill failed
+			if a.err == nil {
+				return errorStreamAbandoned
+			}
+			return a.err
+		}
+		a.cur = b
+	}
+	return nil
+}
+
+// Read will return the next available data.
+func (a *AsyncReader) Read(p []byte) (n int, err error) {
+	a.mu.Lock()
+	defer a.mu.Unlock()
+
+	// Swap buffer and maybe return error
+	err = a.fill()
+	if err != nil {
+		return 0, err
+	}
+
+	// Copy what we can
+	n = copy(p, a.cur.buffer())
+	a.cur.increment(n)
+
+	// If at end of buffer, return any error, if present
+	if a.cur.isEmpty() {
+		a.err = a.cur.err
+		return n, a.err
+	}
+	return n, nil
+}
+
+// WriteTo writes data to w until there's no more data to write or when an error occurs.
+// The return value n is the number of bytes written.
+// Any error encountered during the write is also returned.
+func (a *AsyncReader) WriteTo(w io.Writer) (n int64, err error) {
+	a.mu.Lock()
+	defer a.mu.Unlock()
+
+	n = 0
+	for {
+		err = a.fill()
+		if err != nil {
+			return n, err
+		}
+		n2, err := w.Write(a.cur.buffer())
+		a.cur.increment(n2)
+		n += int64(n2)
+		if err != nil {
+			return n, err
+		}
+		if a.cur.err != nil {
+			a.err = a.cur.err
+			return n, a.cur.err
+		}
+	}
+}
+
+// Abandon will ensure that the underlying async reader is shut down.
+// It will NOT close the input supplied on New.
+func (a *AsyncReader) Abandon() {
+	select {
+	case <-a.exit:
+		// Do nothing if reader routine already exited
+		return
+	default:
+	}
+	// Close and wait for go routine
+	close(a.exit)
+	<-a.exited
+	// take the lock to wait for Read/WriteTo to complete
+	a.mu.Lock()
+	defer a.mu.Unlock()
+	// Return any outstanding buffers to the Pool
+	if a.cur != nil {
+		a.putBuffer(a.cur)
+		a.cur = nil
+	}
+	for b := range a.ready {
+		a.putBuffer(b)
+	}
+}
+
+// Close will ensure that the underlying async reader is shut down.
+// It will also close the input supplied on New.
+func (a *AsyncReader) Close() (err error) {
+	a.Abandon()
+	if a.closed {
+		return nil
+	}
+	a.closed = true
+	return a.in.Close()
+}
+
+// Internal buffer
+// If an error is present, it must be returned
+// once all buffer content has been served.
+type buffer struct {
+	buf    []byte
+	err    error
+	offset int
+}
+
+func newBuffer() *buffer {
+	return &buffer{
+		buf: make([]byte, BufferSize),
+		err: nil,
+	}
+}
+
+// clear returns the buffer to its full size and clears the members
+func (b *buffer) clear() {
+	b.buf = b.buf[:cap(b.buf)]
+	b.err = nil
+	b.offset = 0
+}
+
+// isEmpty returns true is offset is at end of
+// buffer, or
+func (b *buffer) isEmpty() bool {
+	if b == nil {
+		return true
+	}
+	if len(b.buf)-b.offset <= 0 {
+		return true
+	}
+	return false
+}
+
+// read into start of the buffer from the supplied reader,
+// resets the offset and updates the size of the buffer.
+// Any error encountered during the read is returned.
+func (b *buffer) read(rd io.Reader) error {
+	var n int
+	n, b.err = readers.ReadFill(rd, b.buf)
+	b.buf = b.buf[0:n]
+	b.offset = 0
+	return b.err
+}
+
+// Return the buffer at current offset
+func (b *buffer) buffer() []byte {
+	return b.buf[b.offset:]
+}
+
+// increment the offset
+func (b *buffer) increment(n int) {
+	b.offset += n
+}

fs/asyncreader/asyncreader_test.go

@@ -0,0 +1,283 @@
+package asyncreader
+
+import (
+	"bufio"
+	"bytes"
+	"io"
+	"io/ioutil"
+	"strings"
+	"sync"
+	"testing"
+	"testing/iotest"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestAsyncReader(t *testing.T) {
+	buf := ioutil.NopCloser(bytes.NewBufferString("Testbuffer"))
+	ar, err := New(buf, 4)
+	require.NoError(t, err)
+
+	var dst = make([]byte, 100)
+	n, err := ar.Read(dst)
+	assert.Equal(t, io.EOF, err)
+	assert.Equal(t, 10, n)
+
+	n, err = ar.Read(dst)
+	assert.Equal(t, io.EOF, err)
+	assert.Equal(t, 0, n)
+
+	// Test read after error
+	n, err = ar.Read(dst)
+	assert.Equal(t, io.EOF, err)
+	assert.Equal(t, 0, n)
+
+	err = ar.Close()
+	require.NoError(t, err)
+	// Test double close
+	err = ar.Close()
+	require.NoError(t, err)
+
+	// Test Close without reading everything
+	buf = ioutil.NopCloser(bytes.NewBuffer(make([]byte, 50000)))
+	ar, err = New(buf, 4)
+	require.NoError(t, err)
+	err = ar.Close()
+	require.NoError(t, err)
+
+}
+
+func TestAsyncWriteTo(t *testing.T) {
+	buf := ioutil.NopCloser(bytes.NewBufferString("Testbuffer"))
+	ar, err := New(buf, 4)
+	require.NoError(t, err)
+
+	var dst = &bytes.Buffer{}
+	n, err := io.Copy(dst, ar)
+	assert.Equal(t, io.EOF, err)
+	assert.Equal(t, int64(10), n)
+
+	// Should still return EOF
+	n, err = io.Copy(dst, ar)
+	assert.Equal(t, io.EOF, err)
+	assert.Equal(t, int64(0), n)
+
+	err = ar.Close()
+	require.NoError(t, err)
+}
+
+func TestAsyncReaderErrors(t *testing.T) {
+	// test nil reader
+	_, err := New(nil, 4)
+	require.Error(t, err)
+
+	// invalid buffer number
+	buf := ioutil.NopCloser(bytes.NewBufferString("Testbuffer"))
+	_, err = New(buf, 0)
+	require.Error(t, err)
+	_, err = New(buf, -1)
+	require.Error(t, err)
+}
+
+// Complex read tests, leveraged from "bufio".
+
+type readMaker struct {
+	name string
+	fn   func(io.Reader) io.Reader
+}
+
+var readMakers = []readMaker{
+	{"full", func(r io.Reader) io.Reader { return r }},
+	{"byte", iotest.OneByteReader},
+	{"half", iotest.HalfReader},
+	{"data+err", iotest.DataErrReader},
+	{"timeout", iotest.TimeoutReader},
+}
+
+// Call Read to accumulate the text of a file
+func reads(buf io.Reader, m int) string {
+	var b [1000]byte
+	nb := 0
+	for {
+		n, err := buf.Read(b[nb : nb+m])
+		nb += n
+		if err == io.EOF {
+			break
+		} else if err != nil && err != iotest.ErrTimeout {
+			panic("Data: " + err.Error())
+		} else if err != nil {
+			break
+		}
+	}
+	return string(b[0:nb])
+}
+
+type bufReader struct {
+	name string
+	fn   func(io.Reader) string
+}
+
+var bufreaders = []bufReader{
+	{"1", func(b io.Reader) string { return reads(b, 1) }},
+	{"2", func(b io.Reader) string { return reads(b, 2) }},
+	{"3", func(b io.Reader) string { return reads(b, 3) }},
+	{"4", func(b io.Reader) string { return reads(b, 4) }},
+	{"5", func(b io.Reader) string { return reads(b, 5) }},
+	{"7", func(b io.Reader) string { return reads(b, 7) }},
+}
+
+const minReadBufferSize = 16
+
+var bufsizes = []int{
+	0, minReadBufferSize, 23, 32, 46, 64, 93, 128, 1024, 4096,
+}
+
+// Test various  input buffer sizes, number of buffers and read sizes.
+func TestAsyncReaderSizes(t *testing.T) {
+	var texts [31]string
+	str := ""
+	all := ""
+	for i := 0; i < len(texts)-1; i++ {
+		texts[i] = str + "\n"
+		all += texts[i]
+		str += string(i%26 + 'a')
+	}
+	texts[len(texts)-1] = all
+
+	for h := 0; h < len(texts); h++ {
+		text := texts[h]
+		for i := 0; i < len(readMakers); i++ {
+			for j := 0; j < len(bufreaders); j++ {
+				for k := 0; k < len(bufsizes); k++ {
+					for l := 1; l < 10; l++ {
+						readmaker := readMakers[i]
+						bufreader := bufreaders[j]
+						bufsize := bufsizes[k]
+						read := readmaker.fn(strings.NewReader(text))
+						buf := bufio.NewReaderSize(read, bufsize)
+						ar, _ := New(ioutil.NopCloser(buf), l)
+						s := bufreader.fn(ar)
+						// "timeout" expects the Reader to recover, AsyncReader does not.
+						if s != text && readmaker.name != "timeout" {
+							t.Errorf("reader=%s fn=%s bufsize=%d want=%q got=%q",
+								readmaker.name, bufreader.name, bufsize, text, s)
+						}
+						err := ar.Close()
+						require.NoError(t, err)
+					}
+				}
+			}
+		}
+	}
+}
+
+// Test various input buffer sizes, number of buffers and read sizes.
+func TestAsyncReaderWriteTo(t *testing.T) {
+	var texts [31]string
+	str := ""
+	all := ""
+	for i := 0; i < len(texts)-1; i++ {
+		texts[i] = str + "\n"
+		all += texts[i]
+		str += string(i%26 + 'a')
+	}
+	texts[len(texts)-1] = all
+
+	for h := 0; h < len(texts); h++ {
+		text := texts[h]
+		for i := 0; i < len(readMakers); i++ {
+			for j := 0; j < len(bufreaders); j++ {
+				for k := 0; k < len(bufsizes); k++ {
+					for l := 1; l < 10; l++ {
+						readmaker := readMakers[i]
+						bufreader := bufreaders[j]
+						bufsize := bufsizes[k]
+						read := readmaker.fn(strings.NewReader(text))
+						buf := bufio.NewReaderSize(read, bufsize)
+						ar, _ := New(ioutil.NopCloser(buf), l)
+						dst := &bytes.Buffer{}
+						_, err := ar.WriteTo(dst)
+						if err != nil && err != io.EOF && err != iotest.ErrTimeout {
+							t.Fatal("Copy:", err)
+						}
+						s := dst.String()
+						// "timeout" expects the Reader to recover, AsyncReader does not.
+						if s != text && readmaker.name != "timeout" {
+							t.Errorf("reader=%s fn=%s bufsize=%d want=%q got=%q",
+								readmaker.name, bufreader.name, bufsize, text, s)
+						}
+						err = ar.Close()
+						require.NoError(t, err)
+					}
+				}
+			}
+		}
+	}
+}
+
+// Read an infinite number of zeros
+type zeroReader struct {
+	closed bool
+}
+
+func (z *zeroReader) Read(p []byte) (n int, err error) {
+	if z.closed {
+		return 0, io.EOF
+	}
+	for i := range p {
+		p[i] = 0
+	}
+	return len(p), nil
+}
+
+func (z *zeroReader) Close() error {
+	if z.closed {
+		panic("double close on zeroReader")
+	}
+	z.closed = true
+	return nil
+}
+
+// Test closing and abandoning
+func testAsyncReaderClose(t *testing.T, writeto bool) {
+	zr := &zeroReader{}
+	a, err := New(zr, 16)
+	require.NoError(t, err)
+	var copyN int64
+	var copyErr error
+	var wg sync.WaitGroup
+	started := make(chan struct{})
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		close(started)
+		if writeto {
+			// exercise the WriteTo path
+			copyN, copyErr = a.WriteTo(ioutil.Discard)
+		} else {
+			// exercise the Read path
+			buf := make([]byte, 64*1024)
+			for {
+				var n int
+				n, copyErr = a.Read(buf)
+				copyN += int64(n)
+				if copyErr != nil {
+					break
+				}
+			}
+		}
+	}()
+	// Do some copying
+	<-started
+	time.Sleep(100 * time.Millisecond)
+	// Abandon the copy
+	a.Abandon()
+	wg.Wait()
+	assert.Equal(t, errorStreamAbandoned, copyErr)
+	// t.Logf("Copied %d bytes, err %v", copyN, copyErr)
+	assert.True(t, copyN > 0)
+}
+func TestAsyncReaderCloseRead(t *testing.T)    { testAsyncReaderClose(t, false) }
+func TestAsyncReaderCloseWriteTo(t *testing.T) { testAsyncReaderClose(t, true) }