Add context propagation to rclone

- Change rclone/fs interfaces to accept context.Context
- Update interface implementations to use context.Context
- Change top level usage to propagate context to lover level functions

Context propagation is needed for stopping transfers and passing other
request-scoped values.
This commit is contained in:
Aleksandar Jankovic 2019-06-17 10:34:30 +02:00 committed by Nick Craig-Wood
parent a2c317b46e
commit f78cd1e043
156 changed files with 2570 additions and 2380 deletions

View file

@ -4,6 +4,7 @@ package cache_test
import (
"bytes"
"context"
"encoding/base64"
goflag "flag"
"fmt"
@ -120,7 +121,7 @@ func TestInternalListRootAndInnerRemotes(t *testing.T) {
require.NoError(t, err)
listRootInner, err := runInstance.list(t, rootFs, innerFolder)
require.NoError(t, err)
listInner, err := rootFs2.List("")
listInner, err := rootFs2.List(context.Background(), "")
require.NoError(t, err)
require.Len(t, listRoot, 1)
@ -138,10 +139,10 @@ func TestInternalVfsCache(t *testing.T) {
rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, true, true, nil, map[string]string{"writes": "true", "info_age": "1h"})
defer runInstance.cleanupFs(t, rootFs, boltDb)
err := rootFs.Mkdir("test")
err := rootFs.Mkdir(context.Background(), "test")
require.NoError(t, err)
runInstance.writeObjectString(t, rootFs, "test/second", "content")
_, err = rootFs.List("test")
_, err = rootFs.List(context.Background(), "test")
require.NoError(t, err)
testReader := runInstance.randomReader(t, testSize)
@ -266,7 +267,7 @@ func TestInternalObjNotFound(t *testing.T) {
rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, false, true, nil, nil)
defer runInstance.cleanupFs(t, rootFs, boltDb)
obj, err := rootFs.NewObject("404")
obj, err := rootFs.NewObject(context.Background(), "404")
require.Error(t, err)
require.Nil(t, obj)
}
@ -445,7 +446,7 @@ func TestInternalWrappedFsChangeNotSeen(t *testing.T) {
require.NoError(t, err)
log.Printf("original size: %v", originalSize)
o, err := cfs.UnWrap().NewObject(runInstance.encryptRemoteIfNeeded(t, "data.bin"))
o, err := cfs.UnWrap().NewObject(context.Background(), runInstance.encryptRemoteIfNeeded(t, "data.bin"))
require.NoError(t, err)
expectedSize := int64(len([]byte("test content")))
var data2 []byte
@ -457,7 +458,7 @@ func TestInternalWrappedFsChangeNotSeen(t *testing.T) {
data2 = []byte("test content")
}
objInfo := object.NewStaticObjectInfo(runInstance.encryptRemoteIfNeeded(t, "data.bin"), time.Now(), int64(len(data2)), true, nil, cfs.UnWrap())
err = o.Update(bytes.NewReader(data2), objInfo)
err = o.Update(context.Background(), bytes.NewReader(data2), objInfo)
require.NoError(t, err)
require.Equal(t, int64(len(data2)), o.Size())
log.Printf("updated size: %v", len(data2))
@ -503,9 +504,9 @@ func TestInternalMoveWithNotify(t *testing.T) {
} else {
testData = []byte("test content")
}
_ = cfs.UnWrap().Mkdir(runInstance.encryptRemoteIfNeeded(t, "test"))
_ = cfs.UnWrap().Mkdir(runInstance.encryptRemoteIfNeeded(t, "test/one"))
_ = cfs.UnWrap().Mkdir(runInstance.encryptRemoteIfNeeded(t, "test/second"))
_ = cfs.UnWrap().Mkdir(context.Background(), runInstance.encryptRemoteIfNeeded(t, "test"))
_ = cfs.UnWrap().Mkdir(context.Background(), runInstance.encryptRemoteIfNeeded(t, "test/one"))
_ = cfs.UnWrap().Mkdir(context.Background(), runInstance.encryptRemoteIfNeeded(t, "test/second"))
srcObj := runInstance.writeObjectBytes(t, cfs.UnWrap(), srcName, testData)
// list in mount
@ -515,7 +516,7 @@ func TestInternalMoveWithNotify(t *testing.T) {
require.NoError(t, err)
// move file
_, err = cfs.UnWrap().Features().Move(srcObj, dstName)
_, err = cfs.UnWrap().Features().Move(context.Background(), srcObj, dstName)
require.NoError(t, err)
err = runInstance.retryBlock(func() error {
@ -589,9 +590,9 @@ func TestInternalNotifyCreatesEmptyParts(t *testing.T) {
} else {
testData = []byte("test content")
}
err = rootFs.Mkdir("test")
err = rootFs.Mkdir(context.Background(), "test")
require.NoError(t, err)
err = rootFs.Mkdir("test/one")
err = rootFs.Mkdir(context.Background(), "test/one")
require.NoError(t, err)
srcObj := runInstance.writeObjectBytes(t, cfs.UnWrap(), srcName, testData)
@ -608,7 +609,7 @@ func TestInternalNotifyCreatesEmptyParts(t *testing.T) {
require.False(t, found)
// move file
_, err = cfs.UnWrap().Features().Move(srcObj, dstName)
_, err = cfs.UnWrap().Features().Move(context.Background(), srcObj, dstName)
require.NoError(t, err)
err = runInstance.retryBlock(func() error {
@ -670,23 +671,23 @@ func TestInternalChangeSeenAfterDirCacheFlush(t *testing.T) {
runInstance.writeRemoteBytes(t, rootFs, "data.bin", testData)
// update in the wrapped fs
o, err := cfs.UnWrap().NewObject(runInstance.encryptRemoteIfNeeded(t, "data.bin"))
o, err := cfs.UnWrap().NewObject(context.Background(), runInstance.encryptRemoteIfNeeded(t, "data.bin"))
require.NoError(t, err)
wrappedTime := time.Now().Add(-1 * time.Hour)
err = o.SetModTime(wrappedTime)
err = o.SetModTime(context.Background(), wrappedTime)
require.NoError(t, err)
// get a new instance from the cache
co, err := rootFs.NewObject("data.bin")
co, err := rootFs.NewObject(context.Background(), "data.bin")
require.NoError(t, err)
require.NotEqual(t, o.ModTime().String(), co.ModTime().String())
require.NotEqual(t, o.ModTime(context.Background()).String(), co.ModTime(context.Background()).String())
cfs.DirCacheFlush() // flush the cache
// get a new instance from the cache
co, err = rootFs.NewObject("data.bin")
co, err = rootFs.NewObject(context.Background(), "data.bin")
require.NoError(t, err)
require.Equal(t, wrappedTime.Unix(), co.ModTime().Unix())
require.Equal(t, wrappedTime.Unix(), co.ModTime(context.Background()).Unix())
}
func TestInternalChangeSeenAfterRc(t *testing.T) {
@ -713,19 +714,19 @@ func TestInternalChangeSeenAfterRc(t *testing.T) {
runInstance.writeRemoteBytes(t, rootFs, "data.bin", testData)
// update in the wrapped fs
o, err := cfs.UnWrap().NewObject(runInstance.encryptRemoteIfNeeded(t, "data.bin"))
o, err := cfs.UnWrap().NewObject(context.Background(), runInstance.encryptRemoteIfNeeded(t, "data.bin"))
require.NoError(t, err)
wrappedTime := time.Now().Add(-1 * time.Hour)
err = o.SetModTime(wrappedTime)
err = o.SetModTime(context.Background(), wrappedTime)
require.NoError(t, err)
// get a new instance from the cache
co, err := rootFs.NewObject("data.bin")
co, err := rootFs.NewObject(context.Background(), "data.bin")
require.NoError(t, err)
require.NotEqual(t, o.ModTime().String(), co.ModTime().String())
require.NotEqual(t, o.ModTime(context.Background()).String(), co.ModTime(context.Background()).String())
// Call the rc function
m, err := cacheExpire.Fn(rc.Params{"remote": "data.bin"})
m, err := cacheExpire.Fn(context.Background(), rc.Params{"remote": "data.bin"})
require.NoError(t, err)
require.Contains(t, m, "status")
require.Contains(t, m, "message")
@ -733,9 +734,9 @@ func TestInternalChangeSeenAfterRc(t *testing.T) {
require.Contains(t, m["message"], "cached file cleared")
// get a new instance from the cache
co, err = rootFs.NewObject("data.bin")
co, err = rootFs.NewObject(context.Background(), "data.bin")
require.NoError(t, err)
require.Equal(t, wrappedTime.Unix(), co.ModTime().Unix())
require.Equal(t, wrappedTime.Unix(), co.ModTime(context.Background()).Unix())
_, err = runInstance.list(t, rootFs, "")
require.NoError(t, err)
@ -749,7 +750,7 @@ func TestInternalChangeSeenAfterRc(t *testing.T) {
require.Len(t, li1, 1)
// Call the rc function
m, err = cacheExpire.Fn(rc.Params{"remote": "/"})
m, err = cacheExpire.Fn(context.Background(), rc.Params{"remote": "/"})
require.NoError(t, err)
require.Contains(t, m, "status")
require.Contains(t, m, "message")
@ -794,7 +795,7 @@ func TestInternalMaxChunkSizeRespected(t *testing.T) {
// create some rand test data
testData := randStringBytes(int(int64(totalChunks-1)*chunkSize + chunkSize/2))
runInstance.writeRemoteBytes(t, rootFs, "data.bin", testData)
o, err := cfs.NewObject(runInstance.encryptRemoteIfNeeded(t, "data.bin"))
o, err := cfs.NewObject(context.Background(), runInstance.encryptRemoteIfNeeded(t, "data.bin"))
require.NoError(t, err)
co, ok := o.(*cache.Object)
require.True(t, ok)
@ -833,7 +834,7 @@ func TestInternalExpiredEntriesRemoved(t *testing.T) {
require.NoError(t, err)
require.Len(t, l, 1)
err = cfs.UnWrap().Mkdir(runInstance.encryptRemoteIfNeeded(t, "test/third"))
err = cfs.UnWrap().Mkdir(context.Background(), runInstance.encryptRemoteIfNeeded(t, "test/third"))
require.NoError(t, err)
l, err = runInstance.list(t, rootFs, "test")
@ -868,14 +869,14 @@ func TestInternalBug2117(t *testing.T) {
cfs, err := runInstance.getCacheFs(rootFs)
require.NoError(t, err)
err = cfs.UnWrap().Mkdir("test")
err = cfs.UnWrap().Mkdir(context.Background(), "test")
require.NoError(t, err)
for i := 1; i <= 4; i++ {
err = cfs.UnWrap().Mkdir(fmt.Sprintf("test/dir%d", i))
err = cfs.UnWrap().Mkdir(context.Background(), fmt.Sprintf("test/dir%d", i))
require.NoError(t, err)
for j := 1; j <= 4; j++ {
err = cfs.UnWrap().Mkdir(fmt.Sprintf("test/dir%d/dir%d", i, j))
err = cfs.UnWrap().Mkdir(context.Background(), fmt.Sprintf("test/dir%d/dir%d", i, j))
require.NoError(t, err)
runInstance.writeObjectString(t, cfs.UnWrap(), fmt.Sprintf("test/dir%d/dir%d/test.txt", i, j), "test")
@ -1080,10 +1081,10 @@ func (r *run) newCacheFs(t *testing.T, remote, id string, needRemote, purge bool
}
if purge {
_ = f.Features().Purge()
_ = f.Features().Purge(context.Background())
require.NoError(t, err)
}
err = f.Mkdir("")
err = f.Mkdir(context.Background(), "")
require.NoError(t, err)
if r.useMount && !r.isMounted {
r.mountFs(t, f)
@ -1097,7 +1098,7 @@ func (r *run) cleanupFs(t *testing.T, f fs.Fs, b *cache.Persistent) {
r.unmountFs(t, f)
}
err := f.Features().Purge()
err := f.Features().Purge(context.Background())
require.NoError(t, err)
cfs, err := r.getCacheFs(f)
require.NoError(t, err)
@ -1199,7 +1200,7 @@ func (r *run) writeRemoteReader(t *testing.T, f fs.Fs, remote string, in io.Read
func (r *run) writeObjectBytes(t *testing.T, f fs.Fs, remote string, data []byte) fs.Object {
in := bytes.NewReader(data)
_ = r.writeObjectReader(t, f, remote, in)
o, err := f.NewObject(remote)
o, err := f.NewObject(context.Background(), remote)
require.NoError(t, err)
require.Equal(t, int64(len(data)), o.Size())
return o
@ -1208,7 +1209,7 @@ func (r *run) writeObjectBytes(t *testing.T, f fs.Fs, remote string, data []byte
func (r *run) writeObjectReader(t *testing.T, f fs.Fs, remote string, in io.Reader) fs.Object {
modTime := time.Now()
objInfo := object.NewStaticObjectInfo(remote, modTime, -1, true, nil, f)
obj, err := f.Put(in, objInfo)
obj, err := f.Put(context.Background(), in, objInfo)
require.NoError(t, err)
if r.useMount {
r.vfs.WaitForWriters(10 * time.Second)
@ -1228,18 +1229,18 @@ func (r *run) updateObjectRemote(t *testing.T, f fs.Fs, remote string, data1 []b
err = ioutil.WriteFile(path.Join(r.mntDir, remote), data2, 0600)
require.NoError(t, err)
r.vfs.WaitForWriters(10 * time.Second)
obj, err = f.NewObject(remote)
obj, err = f.NewObject(context.Background(), remote)
} else {
in1 := bytes.NewReader(data1)
in2 := bytes.NewReader(data2)
objInfo1 := object.NewStaticObjectInfo(remote, time.Now(), int64(len(data1)), true, nil, f)
objInfo2 := object.NewStaticObjectInfo(remote, time.Now(), int64(len(data2)), true, nil, f)
obj, err = f.Put(in1, objInfo1)
obj, err = f.Put(context.Background(), in1, objInfo1)
require.NoError(t, err)
obj, err = f.NewObject(remote)
obj, err = f.NewObject(context.Background(), remote)
require.NoError(t, err)
err = obj.Update(in2, objInfo2)
err = obj.Update(context.Background(), in2, objInfo2)
}
require.NoError(t, err)
@ -1268,7 +1269,7 @@ func (r *run) readDataFromRemote(t *testing.T, f fs.Fs, remote string, offset, e
return checkSample, err
}
} else {
co, err := f.NewObject(remote)
co, err := f.NewObject(context.Background(), remote)
if err != nil {
return checkSample, err
}
@ -1283,7 +1284,7 @@ func (r *run) readDataFromRemote(t *testing.T, f fs.Fs, remote string, offset, e
func (r *run) readDataFromObj(t *testing.T, o fs.Object, offset, end int64, noLengthCheck bool) []byte {
size := end - offset
checkSample := make([]byte, size)
reader, err := o.Open(&fs.SeekOption{Offset: offset})
reader, err := o.Open(context.Background(), &fs.SeekOption{Offset: offset})
require.NoError(t, err)
totalRead, err := io.ReadFull(reader, checkSample)
if (err == io.EOF || err == io.ErrUnexpectedEOF) && noLengthCheck {
@ -1300,7 +1301,7 @@ func (r *run) mkdir(t *testing.T, f fs.Fs, remote string) {
if r.useMount {
err = os.Mkdir(path.Join(r.mntDir, remote), 0700)
} else {
err = f.Mkdir(remote)
err = f.Mkdir(context.Background(), remote)
}
require.NoError(t, err)
}
@ -1312,11 +1313,11 @@ func (r *run) rm(t *testing.T, f fs.Fs, remote string) error {
err = os.Remove(path.Join(r.mntDir, remote))
} else {
var obj fs.Object
obj, err = f.NewObject(remote)
obj, err = f.NewObject(context.Background(), remote)
if err != nil {
err = f.Rmdir(remote)
err = f.Rmdir(context.Background(), remote)
} else {
err = obj.Remove()
err = obj.Remove(context.Background())
}
}
@ -1334,7 +1335,7 @@ func (r *run) list(t *testing.T, f fs.Fs, remote string) ([]interface{}, error)
}
} else {
var list fs.DirEntries
list, err = f.List(remote)
list, err = f.List(context.Background(), remote)
for _, ll := range list {
l = append(l, ll)
}
@ -1353,7 +1354,7 @@ func (r *run) listPath(t *testing.T, f fs.Fs, remote string) []string {
}
} else {
var list fs.DirEntries
list, err = f.List(remote)
list, err = f.List(context.Background(), remote)
for _, ll := range list {
l = append(l, ll.Remote())
}
@ -1393,7 +1394,7 @@ func (r *run) dirMove(t *testing.T, rootFs fs.Fs, src, dst string) error {
}
r.vfs.WaitForWriters(10 * time.Second)
} else if rootFs.Features().DirMove != nil {
err = rootFs.Features().DirMove(rootFs, src, dst)
err = rootFs.Features().DirMove(context.Background(), rootFs, src, dst)
if err != nil {
return err
}
@ -1415,11 +1416,11 @@ func (r *run) move(t *testing.T, rootFs fs.Fs, src, dst string) error {
}
r.vfs.WaitForWriters(10 * time.Second)
} else if rootFs.Features().Move != nil {
obj1, err := rootFs.NewObject(src)
obj1, err := rootFs.NewObject(context.Background(), src)
if err != nil {
return err
}
_, err = rootFs.Features().Move(obj1, dst)
_, err = rootFs.Features().Move(context.Background(), obj1, dst)
if err != nil {
return err
}
@ -1441,11 +1442,11 @@ func (r *run) copy(t *testing.T, rootFs fs.Fs, src, dst string) error {
}
r.vfs.WaitForWriters(10 * time.Second)
} else if rootFs.Features().Copy != nil {
obj, err := rootFs.NewObject(src)
obj, err := rootFs.NewObject(context.Background(), src)
if err != nil {
return err
}
_, err = rootFs.Features().Copy(obj, dst)
_, err = rootFs.Features().Copy(context.Background(), obj, dst)
if err != nil {
return err
}
@ -1467,11 +1468,11 @@ func (r *run) modTime(t *testing.T, rootFs fs.Fs, src string) (time.Time, error)
}
return fi.ModTime(), nil
}
obj1, err := rootFs.NewObject(src)
obj1, err := rootFs.NewObject(context.Background(), src)
if err != nil {
return time.Time{}, err
}
return obj1.ModTime(), nil
return obj1.ModTime(context.Background()), nil
}
func (r *run) size(t *testing.T, rootFs fs.Fs, src string) (int64, error) {
@ -1484,7 +1485,7 @@ func (r *run) size(t *testing.T, rootFs fs.Fs, src string) (int64, error) {
}
return fi.Size(), nil
}
obj1, err := rootFs.NewObject(src)
obj1, err := rootFs.NewObject(context.Background(), src)
if err != nil {
return int64(0), err
}
@ -1507,14 +1508,14 @@ func (r *run) updateData(t *testing.T, rootFs fs.Fs, src, data, append string) e
_, err = f.WriteString(data + append)
} else {
var obj1 fs.Object
obj1, err = rootFs.NewObject(src)
obj1, err = rootFs.NewObject(context.Background(), src)
if err != nil {
return err
}
data1 := []byte(data + append)
r := bytes.NewReader(data1)
objInfo1 := object.NewStaticObjectInfo(src, time.Now(), int64(len(data1)), true, nil, rootFs)
err = obj1.Update(r, objInfo1)
err = obj1.Update(context.Background(), r, objInfo1)
}
return err