forked from TrueCloudLab/rclone
operations: make Open() return an io.ReadSeekCloser #7350
As part of reducing memory usage in rclone, we need to have a raw handle to an object we can seek with.
This commit is contained in:
Nick Craig-Wood
parent
e8fcde8de1
commit
c0fb9ebfce
4 changed files with 466 additions and 85 deletions
|
|
@ -9,12 +9,17 @@ import (
|
|||
"github.com/rclone/rclone/fs"
|
||||
"github.com/rclone/rclone/fs/hash"
|
||||
"github.com/rclone/rclone/fstest/mockobject"
|
||||
"github.com/rclone/rclone/lib/pool"
|
||||
"github.com/rclone/rclone/lib/readers"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// check interface
|
||||
var _ io.ReadCloser = (*ReOpen)(nil)
|
||||
// check interfaces
|
||||
var (
|
||||
_ io.ReadSeekCloser = (*ReOpen)(nil)
|
||||
_ pool.DelayAccountinger = (*ReOpen)(nil)
|
||||
)
|
||||
|
||||
var errorTestError = errors.New("test error")
|
||||
|
||||
|
|
@ -24,13 +29,36 @@ var errorTestError = errors.New("test error")
|
|||
// error
|
||||
type reOpenTestObject struct {
|
||||
fs.Object
|
||||
breaks []int64
|
||||
t *testing.T
|
||||
breaks []int64
|
||||
unknownSize bool
|
||||
}
|
||||
|
||||
// Open opens the file for read. Call Close() on the returned io.ReadCloser
|
||||
//
|
||||
// This will break after reading the number of bytes in breaks
|
||||
func (o *reOpenTestObject) Open(ctx context.Context, options ...fs.OpenOption) (io.ReadCloser, error) {
|
||||
gotHash := false
|
||||
gotRange := false
|
||||
startPos := int64(0)
|
||||
for _, option := range options {
|
||||
switch x := option.(type) {
|
||||
case *fs.HashesOption:
|
||||
gotHash = true
|
||||
case *fs.RangeOption:
|
||||
gotRange = true
|
||||
startPos = x.Start
|
||||
if o.unknownSize {
|
||||
assert.Equal(o.t, int64(-1), x.End)
|
||||
}
|
||||
case *fs.SeekOption:
|
||||
startPos = x.Offset
|
||||
}
|
||||
}
|
||||
// Check if ranging, mustn't have hash if offset != 0
|
||||
if gotHash && gotRange {
|
||||
assert.Equal(o.t, int64(0), startPos)
|
||||
}
|
||||
rc, err := o.Object.Open(ctx, options...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
|
|
@ -52,28 +80,53 @@ func (o *reOpenTestObject) Open(ctx context.Context, options ...fs.OpenOption) (
|
|||
}
|
||||
|
||||
func TestReOpen(t *testing.T) {
|
||||
for testIndex, testName := range []string{"Seek", "Range"} {
|
||||
for _, testName := range []string{"Normal", "WithRangeOption", "WithSeekOption", "UnknownSize"} {
|
||||
t.Run(testName, func(t *testing.T) {
|
||||
// Contents for the mock object
|
||||
var (
|
||||
reOpenTestcontents = []byte("0123456789")
|
||||
expectedRead = reOpenTestcontents
|
||||
rangeOption *fs.RangeOption
|
||||
seekOption *fs.SeekOption
|
||||
unknownSize = false
|
||||
)
|
||||
if testIndex > 0 {
|
||||
rangeOption = &fs.RangeOption{Start: 1, End: 7}
|
||||
switch testName {
|
||||
case "Normal":
|
||||
case "WithRangeOption":
|
||||
rangeOption = &fs.RangeOption{Start: 1, End: 7} // range is inclusive
|
||||
expectedRead = reOpenTestcontents[1:8]
|
||||
case "WithSeekOption":
|
||||
seekOption = &fs.SeekOption{Offset: 2}
|
||||
expectedRead = reOpenTestcontents[2:]
|
||||
case "UnknownSize":
|
||||
rangeOption = &fs.RangeOption{Start: 1, End: -1}
|
||||
expectedRead = reOpenTestcontents[1:]
|
||||
unknownSize = true
|
||||
default:
|
||||
panic("bad test name")
|
||||
}
|
||||
|
||||
// Start the test with the given breaks
|
||||
testReOpen := func(breaks []int64, maxRetries int) (io.ReadCloser, error) {
|
||||
testReOpen := func(breaks []int64, maxRetries int) (*ReOpen, error) {
|
||||
srcOrig := mockobject.New("potato").WithContent(reOpenTestcontents, mockobject.SeekModeNone)
|
||||
srcOrig.SetUnknownSize(unknownSize)
|
||||
src := &reOpenTestObject{
|
||||
Object: srcOrig,
|
||||
breaks: breaks,
|
||||
Object: srcOrig,
|
||||
t: t,
|
||||
breaks: breaks,
|
||||
unknownSize: unknownSize,
|
||||
}
|
||||
hashOption := &fs.HashesOption{Hashes: hash.NewHashSet(hash.MD5)}
|
||||
return NewReOpen(context.Background(), src, maxRetries, hashOption, rangeOption)
|
||||
opts := []fs.OpenOption{}
|
||||
if rangeOption == nil && seekOption == nil {
|
||||
opts = append(opts, &fs.HashesOption{Hashes: hash.NewHashSet(hash.MD5)})
|
||||
}
|
||||
if rangeOption != nil {
|
||||
opts = append(opts, rangeOption)
|
||||
}
|
||||
if seekOption != nil {
|
||||
opts = append(opts, seekOption)
|
||||
}
|
||||
return NewReOpen(context.Background(), src, maxRetries, opts...)
|
||||
}
|
||||
|
||||
t.Run("Basics", func(t *testing.T) {
|
||||
|
|
@ -92,16 +145,25 @@ func TestReOpen(t *testing.T) {
|
|||
assert.Equal(t, 0, n)
|
||||
assert.Equal(t, io.EOF, err)
|
||||
|
||||
// Rewind the stream
|
||||
_, err = h.Seek(0, io.SeekStart)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Check contents read correctly
|
||||
got, err = io.ReadAll(h)
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, expectedRead, got)
|
||||
|
||||
// Check close
|
||||
assert.NoError(t, h.Close())
|
||||
|
||||
// Check double close
|
||||
assert.Equal(t, errorFileClosed, h.Close())
|
||||
assert.Equal(t, errFileClosed, h.Close())
|
||||
|
||||
// Check read after close
|
||||
n, err = h.Read(buf)
|
||||
assert.Equal(t, 0, n)
|
||||
assert.Equal(t, errorFileClosed, err)
|
||||
assert.Equal(t, errFileClosed, err)
|
||||
})
|
||||
|
||||
t.Run("ErrorAtStart", func(t *testing.T) {
|
||||
|
|
@ -139,10 +201,176 @@ func TestReOpen(t *testing.T) {
|
|||
var buf = make([]byte, 1)
|
||||
n, err := h.Read(buf)
|
||||
assert.Equal(t, 0, n)
|
||||
assert.Equal(t, errorTooManyTries, err)
|
||||
assert.Equal(t, errTooManyTries, err)
|
||||
|
||||
// Check close
|
||||
assert.Equal(t, errorFileClosed, h.Close())
|
||||
assert.Equal(t, errFileClosed, h.Close())
|
||||
})
|
||||
|
||||
t.Run("Seek", func(t *testing.T) {
|
||||
// open
|
||||
h, err := testReOpen([]int64{2, 1, 3}, 10)
|
||||
assert.NoError(t, err)
|
||||
|
||||
// Seek to end
|
||||
pos, err := h.Seek(int64(len(expectedRead)), io.SeekStart)
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, int64(len(expectedRead)), pos)
|
||||
|
||||
// Seek to start
|
||||
pos, err = h.Seek(0, io.SeekStart)
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, int64(0), pos)
|
||||
|
||||
// Should not allow seek past end
|
||||
pos, err = h.Seek(int64(len(expectedRead))+1, io.SeekCurrent)
|
||||
if !unknownSize {
|
||||
assert.Equal(t, errSeekPastEnd, err)
|
||||
assert.Equal(t, len(expectedRead), int(pos))
|
||||
} else {
|
||||
assert.Equal(t, nil, err)
|
||||
assert.Equal(t, len(expectedRead)+1, int(pos))
|
||||
|
||||
// Seek back to start to get tests in sync
|
||||
pos, err = h.Seek(0, io.SeekStart)
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, int64(0), pos)
|
||||
}
|
||||
|
||||
// Should not allow seek to negative position start
|
||||
pos, err = h.Seek(-1, io.SeekCurrent)
|
||||
assert.Equal(t, errNegativeSeek, err)
|
||||
assert.Equal(t, 0, int(pos))
|
||||
|
||||
// Should not allow seek with invalid whence
|
||||
pos, err = h.Seek(0, 3)
|
||||
assert.Equal(t, errInvalidWhence, err)
|
||||
assert.Equal(t, 0, int(pos))
|
||||
|
||||
// check read
|
||||
dst := make([]byte, 5)
|
||||
n, err := h.Read(dst)
|
||||
assert.Nil(t, err)
|
||||
assert.Equal(t, 5, n)
|
||||
assert.Equal(t, expectedRead[:5], dst)
|
||||
|
||||
// Test io.SeekCurrent
|
||||
pos, err = h.Seek(-3, io.SeekCurrent)
|
||||
assert.Nil(t, err)
|
||||
assert.Equal(t, 2, int(pos))
|
||||
|
||||
// check read
|
||||
n, err = h.Read(dst)
|
||||
assert.Nil(t, err)
|
||||
assert.Equal(t, 5, n)
|
||||
assert.Equal(t, expectedRead[2:7], dst)
|
||||
|
||||
pos, err = h.Seek(-2, io.SeekCurrent)
|
||||
assert.Nil(t, err)
|
||||
assert.Equal(t, 5, int(pos))
|
||||
|
||||
// Test io.SeekEnd
|
||||
pos, err = h.Seek(-3, io.SeekEnd)
|
||||
if !unknownSize {
|
||||
assert.Nil(t, err)
|
||||
assert.Equal(t, len(expectedRead)-3, int(pos))
|
||||
} else {
|
||||
assert.Equal(t, errBadEndSeek, err)
|
||||
assert.Equal(t, 0, int(pos))
|
||||
|
||||
// sync
|
||||
pos, err = h.Seek(1, io.SeekCurrent)
|
||||
assert.Nil(t, err)
|
||||
assert.Equal(t, 6, int(pos))
|
||||
}
|
||||
|
||||
// check read
|
||||
dst = make([]byte, 3)
|
||||
n, err = h.Read(dst)
|
||||
assert.Nil(t, err)
|
||||
assert.Equal(t, 3, n)
|
||||
assert.Equal(t, expectedRead[len(expectedRead)-3:], dst)
|
||||
|
||||
// check close
|
||||
assert.NoError(t, h.Close())
|
||||
_, err = h.Seek(0, io.SeekCurrent)
|
||||
assert.Equal(t, errFileClosed, err)
|
||||
})
|
||||
|
||||
t.Run("AccountRead", func(t *testing.T) {
|
||||
h, err := testReOpen(nil, 10)
|
||||
assert.NoError(t, err)
|
||||
|
||||
var total int
|
||||
h.SetAccounting(func(n int) error {
|
||||
total += n
|
||||
return nil
|
||||
})
|
||||
|
||||
dst := make([]byte, 3)
|
||||
n, err := h.Read(dst)
|
||||
assert.Equal(t, 3, n)
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, 3, total)
|
||||
})
|
||||
|
||||
t.Run("AccountReadDelay", func(t *testing.T) {
|
||||
h, err := testReOpen(nil, 10)
|
||||
assert.NoError(t, err)
|
||||
|
||||
var total int
|
||||
h.SetAccounting(func(n int) error {
|
||||
total += n
|
||||
return nil
|
||||
})
|
||||
|
||||
rewind := func() {
|
||||
_, err := h.Seek(0, io.SeekStart)
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
h.DelayAccounting(3)
|
||||
|
||||
dst := make([]byte, 16)
|
||||
|
||||
n, err := h.Read(dst)
|
||||
assert.Equal(t, len(expectedRead), n)
|
||||
assert.Equal(t, io.EOF, err)
|
||||
assert.Equal(t, 0, total)
|
||||
rewind()
|
||||
|
||||
n, err = h.Read(dst)
|
||||
assert.Equal(t, len(expectedRead), n)
|
||||
assert.Equal(t, io.EOF, err)
|
||||
assert.Equal(t, 0, total)
|
||||
rewind()
|
||||
|
||||
n, err = h.Read(dst)
|
||||
assert.Equal(t, len(expectedRead), n)
|
||||
assert.Equal(t, io.EOF, err)
|
||||
assert.Equal(t, len(expectedRead), total)
|
||||
rewind()
|
||||
|
||||
n, err = h.Read(dst)
|
||||
assert.Equal(t, len(expectedRead), n)
|
||||
assert.Equal(t, io.EOF, err)
|
||||
assert.Equal(t, 2*len(expectedRead), total)
|
||||
rewind()
|
||||
})
|
||||
|
||||
t.Run("AccountReadError", func(t *testing.T) {
|
||||
// Test accounting errors
|
||||
h, err := testReOpen(nil, 10)
|
||||
assert.NoError(t, err)
|
||||
|
||||
h.SetAccounting(func(n int) error {
|
||||
return errorTestError
|
||||
})
|
||||
|
||||
dst := make([]byte, 3)
|
||||
n, err := h.Read(dst)
|
||||
assert.Equal(t, 3, n)
|
||||
assert.Equal(t, errorTestError, err)
|
||||
})
|
||||
})
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue