Add context propagation to rclone
- Change rclone/fs interfaces to accept context.Context - Update interface implementations to use context.Context - Change top level usage to propagate context to lover level functions Context propagation is needed for stopping transfers and passing other request-scoped values.
This commit is contained in:
parent
a2c317b46e
commit
f78cd1e043
156 changed files with 2570 additions and 2380 deletions
119
backend/cache/cache_internal_test.go
vendored
119
backend/cache/cache_internal_test.go
vendored
|
@ -4,6 +4,7 @@ package cache_test
|
|||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/base64"
|
||||
goflag "flag"
|
||||
"fmt"
|
||||
|
@ -120,7 +121,7 @@ func TestInternalListRootAndInnerRemotes(t *testing.T) {
|
|||
require.NoError(t, err)
|
||||
listRootInner, err := runInstance.list(t, rootFs, innerFolder)
|
||||
require.NoError(t, err)
|
||||
listInner, err := rootFs2.List("")
|
||||
listInner, err := rootFs2.List(context.Background(), "")
|
||||
require.NoError(t, err)
|
||||
|
||||
require.Len(t, listRoot, 1)
|
||||
|
@ -138,10 +139,10 @@ func TestInternalVfsCache(t *testing.T) {
|
|||
rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, true, true, nil, map[string]string{"writes": "true", "info_age": "1h"})
|
||||
defer runInstance.cleanupFs(t, rootFs, boltDb)
|
||||
|
||||
err := rootFs.Mkdir("test")
|
||||
err := rootFs.Mkdir(context.Background(), "test")
|
||||
require.NoError(t, err)
|
||||
runInstance.writeObjectString(t, rootFs, "test/second", "content")
|
||||
_, err = rootFs.List("test")
|
||||
_, err = rootFs.List(context.Background(), "test")
|
||||
require.NoError(t, err)
|
||||
|
||||
testReader := runInstance.randomReader(t, testSize)
|
||||
|
@ -266,7 +267,7 @@ func TestInternalObjNotFound(t *testing.T) {
|
|||
rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, false, true, nil, nil)
|
||||
defer runInstance.cleanupFs(t, rootFs, boltDb)
|
||||
|
||||
obj, err := rootFs.NewObject("404")
|
||||
obj, err := rootFs.NewObject(context.Background(), "404")
|
||||
require.Error(t, err)
|
||||
require.Nil(t, obj)
|
||||
}
|
||||
|
@ -445,7 +446,7 @@ func TestInternalWrappedFsChangeNotSeen(t *testing.T) {
|
|||
require.NoError(t, err)
|
||||
log.Printf("original size: %v", originalSize)
|
||||
|
||||
o, err := cfs.UnWrap().NewObject(runInstance.encryptRemoteIfNeeded(t, "data.bin"))
|
||||
o, err := cfs.UnWrap().NewObject(context.Background(), runInstance.encryptRemoteIfNeeded(t, "data.bin"))
|
||||
require.NoError(t, err)
|
||||
expectedSize := int64(len([]byte("test content")))
|
||||
var data2 []byte
|
||||
|
@ -457,7 +458,7 @@ func TestInternalWrappedFsChangeNotSeen(t *testing.T) {
|
|||
data2 = []byte("test content")
|
||||
}
|
||||
objInfo := object.NewStaticObjectInfo(runInstance.encryptRemoteIfNeeded(t, "data.bin"), time.Now(), int64(len(data2)), true, nil, cfs.UnWrap())
|
||||
err = o.Update(bytes.NewReader(data2), objInfo)
|
||||
err = o.Update(context.Background(), bytes.NewReader(data2), objInfo)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, int64(len(data2)), o.Size())
|
||||
log.Printf("updated size: %v", len(data2))
|
||||
|
@ -503,9 +504,9 @@ func TestInternalMoveWithNotify(t *testing.T) {
|
|||
} else {
|
||||
testData = []byte("test content")
|
||||
}
|
||||
_ = cfs.UnWrap().Mkdir(runInstance.encryptRemoteIfNeeded(t, "test"))
|
||||
_ = cfs.UnWrap().Mkdir(runInstance.encryptRemoteIfNeeded(t, "test/one"))
|
||||
_ = cfs.UnWrap().Mkdir(runInstance.encryptRemoteIfNeeded(t, "test/second"))
|
||||
_ = cfs.UnWrap().Mkdir(context.Background(), runInstance.encryptRemoteIfNeeded(t, "test"))
|
||||
_ = cfs.UnWrap().Mkdir(context.Background(), runInstance.encryptRemoteIfNeeded(t, "test/one"))
|
||||
_ = cfs.UnWrap().Mkdir(context.Background(), runInstance.encryptRemoteIfNeeded(t, "test/second"))
|
||||
srcObj := runInstance.writeObjectBytes(t, cfs.UnWrap(), srcName, testData)
|
||||
|
||||
// list in mount
|
||||
|
@ -515,7 +516,7 @@ func TestInternalMoveWithNotify(t *testing.T) {
|
|||
require.NoError(t, err)
|
||||
|
||||
// move file
|
||||
_, err = cfs.UnWrap().Features().Move(srcObj, dstName)
|
||||
_, err = cfs.UnWrap().Features().Move(context.Background(), srcObj, dstName)
|
||||
require.NoError(t, err)
|
||||
|
||||
err = runInstance.retryBlock(func() error {
|
||||
|
@ -589,9 +590,9 @@ func TestInternalNotifyCreatesEmptyParts(t *testing.T) {
|
|||
} else {
|
||||
testData = []byte("test content")
|
||||
}
|
||||
err = rootFs.Mkdir("test")
|
||||
err = rootFs.Mkdir(context.Background(), "test")
|
||||
require.NoError(t, err)
|
||||
err = rootFs.Mkdir("test/one")
|
||||
err = rootFs.Mkdir(context.Background(), "test/one")
|
||||
require.NoError(t, err)
|
||||
srcObj := runInstance.writeObjectBytes(t, cfs.UnWrap(), srcName, testData)
|
||||
|
||||
|
@ -608,7 +609,7 @@ func TestInternalNotifyCreatesEmptyParts(t *testing.T) {
|
|||
require.False(t, found)
|
||||
|
||||
// move file
|
||||
_, err = cfs.UnWrap().Features().Move(srcObj, dstName)
|
||||
_, err = cfs.UnWrap().Features().Move(context.Background(), srcObj, dstName)
|
||||
require.NoError(t, err)
|
||||
|
||||
err = runInstance.retryBlock(func() error {
|
||||
|
@ -670,23 +671,23 @@ func TestInternalChangeSeenAfterDirCacheFlush(t *testing.T) {
|
|||
runInstance.writeRemoteBytes(t, rootFs, "data.bin", testData)
|
||||
|
||||
// update in the wrapped fs
|
||||
o, err := cfs.UnWrap().NewObject(runInstance.encryptRemoteIfNeeded(t, "data.bin"))
|
||||
o, err := cfs.UnWrap().NewObject(context.Background(), runInstance.encryptRemoteIfNeeded(t, "data.bin"))
|
||||
require.NoError(t, err)
|
||||
wrappedTime := time.Now().Add(-1 * time.Hour)
|
||||
err = o.SetModTime(wrappedTime)
|
||||
err = o.SetModTime(context.Background(), wrappedTime)
|
||||
require.NoError(t, err)
|
||||
|
||||
// get a new instance from the cache
|
||||
co, err := rootFs.NewObject("data.bin")
|
||||
co, err := rootFs.NewObject(context.Background(), "data.bin")
|
||||
require.NoError(t, err)
|
||||
require.NotEqual(t, o.ModTime().String(), co.ModTime().String())
|
||||
require.NotEqual(t, o.ModTime(context.Background()).String(), co.ModTime(context.Background()).String())
|
||||
|
||||
cfs.DirCacheFlush() // flush the cache
|
||||
|
||||
// get a new instance from the cache
|
||||
co, err = rootFs.NewObject("data.bin")
|
||||
co, err = rootFs.NewObject(context.Background(), "data.bin")
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, wrappedTime.Unix(), co.ModTime().Unix())
|
||||
require.Equal(t, wrappedTime.Unix(), co.ModTime(context.Background()).Unix())
|
||||
}
|
||||
|
||||
func TestInternalChangeSeenAfterRc(t *testing.T) {
|
||||
|
@ -713,19 +714,19 @@ func TestInternalChangeSeenAfterRc(t *testing.T) {
|
|||
runInstance.writeRemoteBytes(t, rootFs, "data.bin", testData)
|
||||
|
||||
// update in the wrapped fs
|
||||
o, err := cfs.UnWrap().NewObject(runInstance.encryptRemoteIfNeeded(t, "data.bin"))
|
||||
o, err := cfs.UnWrap().NewObject(context.Background(), runInstance.encryptRemoteIfNeeded(t, "data.bin"))
|
||||
require.NoError(t, err)
|
||||
wrappedTime := time.Now().Add(-1 * time.Hour)
|
||||
err = o.SetModTime(wrappedTime)
|
||||
err = o.SetModTime(context.Background(), wrappedTime)
|
||||
require.NoError(t, err)
|
||||
|
||||
// get a new instance from the cache
|
||||
co, err := rootFs.NewObject("data.bin")
|
||||
co, err := rootFs.NewObject(context.Background(), "data.bin")
|
||||
require.NoError(t, err)
|
||||
require.NotEqual(t, o.ModTime().String(), co.ModTime().String())
|
||||
require.NotEqual(t, o.ModTime(context.Background()).String(), co.ModTime(context.Background()).String())
|
||||
|
||||
// Call the rc function
|
||||
m, err := cacheExpire.Fn(rc.Params{"remote": "data.bin"})
|
||||
m, err := cacheExpire.Fn(context.Background(), rc.Params{"remote": "data.bin"})
|
||||
require.NoError(t, err)
|
||||
require.Contains(t, m, "status")
|
||||
require.Contains(t, m, "message")
|
||||
|
@ -733,9 +734,9 @@ func TestInternalChangeSeenAfterRc(t *testing.T) {
|
|||
require.Contains(t, m["message"], "cached file cleared")
|
||||
|
||||
// get a new instance from the cache
|
||||
co, err = rootFs.NewObject("data.bin")
|
||||
co, err = rootFs.NewObject(context.Background(), "data.bin")
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, wrappedTime.Unix(), co.ModTime().Unix())
|
||||
require.Equal(t, wrappedTime.Unix(), co.ModTime(context.Background()).Unix())
|
||||
_, err = runInstance.list(t, rootFs, "")
|
||||
require.NoError(t, err)
|
||||
|
||||
|
@ -749,7 +750,7 @@ func TestInternalChangeSeenAfterRc(t *testing.T) {
|
|||
require.Len(t, li1, 1)
|
||||
|
||||
// Call the rc function
|
||||
m, err = cacheExpire.Fn(rc.Params{"remote": "/"})
|
||||
m, err = cacheExpire.Fn(context.Background(), rc.Params{"remote": "/"})
|
||||
require.NoError(t, err)
|
||||
require.Contains(t, m, "status")
|
||||
require.Contains(t, m, "message")
|
||||
|
@ -794,7 +795,7 @@ func TestInternalMaxChunkSizeRespected(t *testing.T) {
|
|||
// create some rand test data
|
||||
testData := randStringBytes(int(int64(totalChunks-1)*chunkSize + chunkSize/2))
|
||||
runInstance.writeRemoteBytes(t, rootFs, "data.bin", testData)
|
||||
o, err := cfs.NewObject(runInstance.encryptRemoteIfNeeded(t, "data.bin"))
|
||||
o, err := cfs.NewObject(context.Background(), runInstance.encryptRemoteIfNeeded(t, "data.bin"))
|
||||
require.NoError(t, err)
|
||||
co, ok := o.(*cache.Object)
|
||||
require.True(t, ok)
|
||||
|
@ -833,7 +834,7 @@ func TestInternalExpiredEntriesRemoved(t *testing.T) {
|
|||
require.NoError(t, err)
|
||||
require.Len(t, l, 1)
|
||||
|
||||
err = cfs.UnWrap().Mkdir(runInstance.encryptRemoteIfNeeded(t, "test/third"))
|
||||
err = cfs.UnWrap().Mkdir(context.Background(), runInstance.encryptRemoteIfNeeded(t, "test/third"))
|
||||
require.NoError(t, err)
|
||||
|
||||
l, err = runInstance.list(t, rootFs, "test")
|
||||
|
@ -868,14 +869,14 @@ func TestInternalBug2117(t *testing.T) {
|
|||
cfs, err := runInstance.getCacheFs(rootFs)
|
||||
require.NoError(t, err)
|
||||
|
||||
err = cfs.UnWrap().Mkdir("test")
|
||||
err = cfs.UnWrap().Mkdir(context.Background(), "test")
|
||||
require.NoError(t, err)
|
||||
for i := 1; i <= 4; i++ {
|
||||
err = cfs.UnWrap().Mkdir(fmt.Sprintf("test/dir%d", i))
|
||||
err = cfs.UnWrap().Mkdir(context.Background(), fmt.Sprintf("test/dir%d", i))
|
||||
require.NoError(t, err)
|
||||
|
||||
for j := 1; j <= 4; j++ {
|
||||
err = cfs.UnWrap().Mkdir(fmt.Sprintf("test/dir%d/dir%d", i, j))
|
||||
err = cfs.UnWrap().Mkdir(context.Background(), fmt.Sprintf("test/dir%d/dir%d", i, j))
|
||||
require.NoError(t, err)
|
||||
|
||||
runInstance.writeObjectString(t, cfs.UnWrap(), fmt.Sprintf("test/dir%d/dir%d/test.txt", i, j), "test")
|
||||
|
@ -1080,10 +1081,10 @@ func (r *run) newCacheFs(t *testing.T, remote, id string, needRemote, purge bool
|
|||
}
|
||||
|
||||
if purge {
|
||||
_ = f.Features().Purge()
|
||||
_ = f.Features().Purge(context.Background())
|
||||
require.NoError(t, err)
|
||||
}
|
||||
err = f.Mkdir("")
|
||||
err = f.Mkdir(context.Background(), "")
|
||||
require.NoError(t, err)
|
||||
if r.useMount && !r.isMounted {
|
||||
r.mountFs(t, f)
|
||||
|
@ -1097,7 +1098,7 @@ func (r *run) cleanupFs(t *testing.T, f fs.Fs, b *cache.Persistent) {
|
|||
r.unmountFs(t, f)
|
||||
}
|
||||
|
||||
err := f.Features().Purge()
|
||||
err := f.Features().Purge(context.Background())
|
||||
require.NoError(t, err)
|
||||
cfs, err := r.getCacheFs(f)
|
||||
require.NoError(t, err)
|
||||
|
@ -1199,7 +1200,7 @@ func (r *run) writeRemoteReader(t *testing.T, f fs.Fs, remote string, in io.Read
|
|||
func (r *run) writeObjectBytes(t *testing.T, f fs.Fs, remote string, data []byte) fs.Object {
|
||||
in := bytes.NewReader(data)
|
||||
_ = r.writeObjectReader(t, f, remote, in)
|
||||
o, err := f.NewObject(remote)
|
||||
o, err := f.NewObject(context.Background(), remote)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, int64(len(data)), o.Size())
|
||||
return o
|
||||
|
@ -1208,7 +1209,7 @@ func (r *run) writeObjectBytes(t *testing.T, f fs.Fs, remote string, data []byte
|
|||
func (r *run) writeObjectReader(t *testing.T, f fs.Fs, remote string, in io.Reader) fs.Object {
|
||||
modTime := time.Now()
|
||||
objInfo := object.NewStaticObjectInfo(remote, modTime, -1, true, nil, f)
|
||||
obj, err := f.Put(in, objInfo)
|
||||
obj, err := f.Put(context.Background(), in, objInfo)
|
||||
require.NoError(t, err)
|
||||
if r.useMount {
|
||||
r.vfs.WaitForWriters(10 * time.Second)
|
||||
|
@ -1228,18 +1229,18 @@ func (r *run) updateObjectRemote(t *testing.T, f fs.Fs, remote string, data1 []b
|
|||
err = ioutil.WriteFile(path.Join(r.mntDir, remote), data2, 0600)
|
||||
require.NoError(t, err)
|
||||
r.vfs.WaitForWriters(10 * time.Second)
|
||||
obj, err = f.NewObject(remote)
|
||||
obj, err = f.NewObject(context.Background(), remote)
|
||||
} else {
|
||||
in1 := bytes.NewReader(data1)
|
||||
in2 := bytes.NewReader(data2)
|
||||
objInfo1 := object.NewStaticObjectInfo(remote, time.Now(), int64(len(data1)), true, nil, f)
|
||||
objInfo2 := object.NewStaticObjectInfo(remote, time.Now(), int64(len(data2)), true, nil, f)
|
||||
|
||||
obj, err = f.Put(in1, objInfo1)
|
||||
obj, err = f.Put(context.Background(), in1, objInfo1)
|
||||
require.NoError(t, err)
|
||||
obj, err = f.NewObject(remote)
|
||||
obj, err = f.NewObject(context.Background(), remote)
|
||||
require.NoError(t, err)
|
||||
err = obj.Update(in2, objInfo2)
|
||||
err = obj.Update(context.Background(), in2, objInfo2)
|
||||
}
|
||||
require.NoError(t, err)
|
||||
|
||||
|
@ -1268,7 +1269,7 @@ func (r *run) readDataFromRemote(t *testing.T, f fs.Fs, remote string, offset, e
|
|||
return checkSample, err
|
||||
}
|
||||
} else {
|
||||
co, err := f.NewObject(remote)
|
||||
co, err := f.NewObject(context.Background(), remote)
|
||||
if err != nil {
|
||||
return checkSample, err
|
||||
}
|
||||
|
@ -1283,7 +1284,7 @@ func (r *run) readDataFromRemote(t *testing.T, f fs.Fs, remote string, offset, e
|
|||
func (r *run) readDataFromObj(t *testing.T, o fs.Object, offset, end int64, noLengthCheck bool) []byte {
|
||||
size := end - offset
|
||||
checkSample := make([]byte, size)
|
||||
reader, err := o.Open(&fs.SeekOption{Offset: offset})
|
||||
reader, err := o.Open(context.Background(), &fs.SeekOption{Offset: offset})
|
||||
require.NoError(t, err)
|
||||
totalRead, err := io.ReadFull(reader, checkSample)
|
||||
if (err == io.EOF || err == io.ErrUnexpectedEOF) && noLengthCheck {
|
||||
|
@ -1300,7 +1301,7 @@ func (r *run) mkdir(t *testing.T, f fs.Fs, remote string) {
|
|||
if r.useMount {
|
||||
err = os.Mkdir(path.Join(r.mntDir, remote), 0700)
|
||||
} else {
|
||||
err = f.Mkdir(remote)
|
||||
err = f.Mkdir(context.Background(), remote)
|
||||
}
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
@ -1312,11 +1313,11 @@ func (r *run) rm(t *testing.T, f fs.Fs, remote string) error {
|
|||
err = os.Remove(path.Join(r.mntDir, remote))
|
||||
} else {
|
||||
var obj fs.Object
|
||||
obj, err = f.NewObject(remote)
|
||||
obj, err = f.NewObject(context.Background(), remote)
|
||||
if err != nil {
|
||||
err = f.Rmdir(remote)
|
||||
err = f.Rmdir(context.Background(), remote)
|
||||
} else {
|
||||
err = obj.Remove()
|
||||
err = obj.Remove(context.Background())
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -1334,7 +1335,7 @@ func (r *run) list(t *testing.T, f fs.Fs, remote string) ([]interface{}, error)
|
|||
}
|
||||
} else {
|
||||
var list fs.DirEntries
|
||||
list, err = f.List(remote)
|
||||
list, err = f.List(context.Background(), remote)
|
||||
for _, ll := range list {
|
||||
l = append(l, ll)
|
||||
}
|
||||
|
@ -1353,7 +1354,7 @@ func (r *run) listPath(t *testing.T, f fs.Fs, remote string) []string {
|
|||
}
|
||||
} else {
|
||||
var list fs.DirEntries
|
||||
list, err = f.List(remote)
|
||||
list, err = f.List(context.Background(), remote)
|
||||
for _, ll := range list {
|
||||
l = append(l, ll.Remote())
|
||||
}
|
||||
|
@ -1393,7 +1394,7 @@ func (r *run) dirMove(t *testing.T, rootFs fs.Fs, src, dst string) error {
|
|||
}
|
||||
r.vfs.WaitForWriters(10 * time.Second)
|
||||
} else if rootFs.Features().DirMove != nil {
|
||||
err = rootFs.Features().DirMove(rootFs, src, dst)
|
||||
err = rootFs.Features().DirMove(context.Background(), rootFs, src, dst)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
@ -1415,11 +1416,11 @@ func (r *run) move(t *testing.T, rootFs fs.Fs, src, dst string) error {
|
|||
}
|
||||
r.vfs.WaitForWriters(10 * time.Second)
|
||||
} else if rootFs.Features().Move != nil {
|
||||
obj1, err := rootFs.NewObject(src)
|
||||
obj1, err := rootFs.NewObject(context.Background(), src)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = rootFs.Features().Move(obj1, dst)
|
||||
_, err = rootFs.Features().Move(context.Background(), obj1, dst)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
@ -1441,11 +1442,11 @@ func (r *run) copy(t *testing.T, rootFs fs.Fs, src, dst string) error {
|
|||
}
|
||||
r.vfs.WaitForWriters(10 * time.Second)
|
||||
} else if rootFs.Features().Copy != nil {
|
||||
obj, err := rootFs.NewObject(src)
|
||||
obj, err := rootFs.NewObject(context.Background(), src)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = rootFs.Features().Copy(obj, dst)
|
||||
_, err = rootFs.Features().Copy(context.Background(), obj, dst)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
@ -1467,11 +1468,11 @@ func (r *run) modTime(t *testing.T, rootFs fs.Fs, src string) (time.Time, error)
|
|||
}
|
||||
return fi.ModTime(), nil
|
||||
}
|
||||
obj1, err := rootFs.NewObject(src)
|
||||
obj1, err := rootFs.NewObject(context.Background(), src)
|
||||
if err != nil {
|
||||
return time.Time{}, err
|
||||
}
|
||||
return obj1.ModTime(), nil
|
||||
return obj1.ModTime(context.Background()), nil
|
||||
}
|
||||
|
||||
func (r *run) size(t *testing.T, rootFs fs.Fs, src string) (int64, error) {
|
||||
|
@ -1484,7 +1485,7 @@ func (r *run) size(t *testing.T, rootFs fs.Fs, src string) (int64, error) {
|
|||
}
|
||||
return fi.Size(), nil
|
||||
}
|
||||
obj1, err := rootFs.NewObject(src)
|
||||
obj1, err := rootFs.NewObject(context.Background(), src)
|
||||
if err != nil {
|
||||
return int64(0), err
|
||||
}
|
||||
|
@ -1507,14 +1508,14 @@ func (r *run) updateData(t *testing.T, rootFs fs.Fs, src, data, append string) e
|
|||
_, err = f.WriteString(data + append)
|
||||
} else {
|
||||
var obj1 fs.Object
|
||||
obj1, err = rootFs.NewObject(src)
|
||||
obj1, err = rootFs.NewObject(context.Background(), src)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
data1 := []byte(data + append)
|
||||
r := bytes.NewReader(data1)
|
||||
objInfo1 := object.NewStaticObjectInfo(src, time.Now(), int64(len(data1)), true, nil, rootFs)
|
||||
err = obj1.Update(r, objInfo1)
|
||||
err = obj1.Update(context.Background(), r, objInfo1)
|
||||
}
|
||||
|
||||
return err
|
||||
|
|
Loading…
Add table
Add a link
Reference in a new issue