rclone/fs/walk_test.go
Nick Craig-Wood 50928a5027 Implement --fast-list flag.
This is supported remotes which can do a recursive listing.  It will
use more memory.

This is related to #1277 but doesn't fix that issue yet.
2017-06-14 16:49:40 +01:00

545 lines
11 KiB
Go

package fs
import (
"fmt"
"sync"
"testing"
"github.com/pkg/errors"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
type (
listResult struct {
entries DirEntries
err error
}
listResults map[string]listResult
errorMap map[string]error
listDirs struct {
mu sync.Mutex
t *testing.T
fs Fs
includeAll bool
results listResults
walkResults listResults
walkErrors errorMap
finalError error
checkMaps bool
maxLevel int
}
)
func newListDirs(t *testing.T, f Fs, includeAll bool, results listResults, walkErrors errorMap, finalError error) *listDirs {
return &listDirs{
t: t,
fs: f,
includeAll: includeAll,
results: results,
walkErrors: walkErrors,
walkResults: listResults{},
finalError: finalError,
checkMaps: true,
maxLevel: -1,
}
}
// NoCheckMaps marks the maps as to be ignored at the end
func (ls *listDirs) NoCheckMaps() *listDirs {
ls.checkMaps = false
return ls
}
// SetLevel(1) turns off recursion
func (ls *listDirs) SetLevel(maxLevel int) *listDirs {
ls.maxLevel = maxLevel
return ls
}
// ListDir returns the expected listing for the directory
func (ls *listDirs) ListDir(f Fs, includeAll bool, dir string) (entries DirEntries, err error) {
ls.mu.Lock()
defer ls.mu.Unlock()
assert.Equal(ls.t, ls.fs, f)
assert.Equal(ls.t, ls.includeAll, includeAll)
// Fetch results for this path
result, ok := ls.results[dir]
if !ok {
ls.t.Errorf("Unexpected list of %q", dir)
return nil, errors.New("unexpected list")
}
delete(ls.results, dir)
// Put expected results for call of WalkFn
ls.walkResults[dir] = result
return result.entries, result.err
}
// ListR returns the expected listing for the directory using ListR
func (ls *listDirs) ListR(f Fs, dir string, callback listRCallback) (err error) {
ls.mu.Lock()
defer ls.mu.Unlock()
assert.Equal(ls.t, ls.fs, f)
//assert.Equal(ls.t, ls.includeAll, includeAll)
var errorReturn error
for dirPath, result := range ls.results {
// Put expected results for call of WalkFn
// Note that we don't call the function at all if we got an error
if result.err != nil {
errorReturn = result.err
}
if errorReturn == nil {
err = callback(result.entries)
require.NoError(ls.t, err)
ls.walkResults[dirPath] = result
}
}
ls.results = listResults{}
return errorReturn
}
// IsFinished checks everything expected was used up
func (ls *listDirs) IsFinished() {
if ls.checkMaps {
assert.Equal(ls.t, errorMap{}, ls.walkErrors)
assert.Equal(ls.t, listResults{}, ls.results)
assert.Equal(ls.t, listResults{}, ls.walkResults)
}
}
// WalkFn is called by the walk to test the expectations
func (ls *listDirs) WalkFn(dir string, entries DirEntries, err error) error {
ls.mu.Lock()
defer ls.mu.Unlock()
// ls.t.Logf("WalkFn(%q, %v, %q)", dir, entries, err)
// Fetch expected entries and err
result, ok := ls.walkResults[dir]
if !ok {
ls.t.Errorf("Unexpected walk of %q (result not found)", dir)
return errors.New("result not found")
}
delete(ls.walkResults, dir)
// Check arguments are as expected
assert.Equal(ls.t, result.entries, entries)
assert.Equal(ls.t, result.err, err)
// Fetch return value
returnErr, ok := ls.walkErrors[dir]
if !ok {
ls.t.Errorf("Unexpected walk of %q (error not found)", dir)
return errors.New("error not found")
}
delete(ls.walkErrors, dir)
return returnErr
}
// Walk does the walk and tests the expectations
func (ls *listDirs) Walk() {
err := walk(nil, "", ls.includeAll, ls.maxLevel, ls.WalkFn, ls.ListDir)
assert.Equal(ls.t, ls.finalError, err)
ls.IsFinished()
}
// WalkR does the walkR and tests the expectations
func (ls *listDirs) WalkR() {
err := walkR(nil, "", ls.includeAll, ls.maxLevel, ls.WalkFn, ls.ListR)
assert.Equal(ls.t, ls.finalError, err)
if ls.finalError == nil {
ls.IsFinished()
}
}
func newDir(name string) *Dir {
return &Dir{Name: name}
}
func testWalkEmpty(t *testing.T) *listDirs {
return newListDirs(t, nil, false,
listResults{
"": {entries: DirEntries{}, err: nil},
},
errorMap{
"": nil,
},
nil,
)
}
func TestWalkEmpty(t *testing.T) { testWalkEmpty(t).Walk() }
func TestWalkREmpty(t *testing.T) { testWalkEmpty(t).WalkR() }
func testWalkEmptySkip(t *testing.T) *listDirs {
return newListDirs(t, nil, true,
listResults{
"": {entries: DirEntries{}, err: nil},
},
errorMap{
"": ErrorSkipDir,
},
nil,
)
}
func TestWalkEmptySkip(t *testing.T) { testWalkEmptySkip(t).Walk() }
func TestWalkREmptySkip(t *testing.T) { testWalkEmptySkip(t).WalkR() }
func testWalkNotFound(t *testing.T) *listDirs {
return newListDirs(t, nil, true,
listResults{
"": {err: ErrorDirNotFound},
},
errorMap{
"": ErrorDirNotFound,
},
ErrorDirNotFound,
)
}
func TestWalkNotFound(t *testing.T) { testWalkNotFound(t).Walk() }
func TestWalkRNotFound(t *testing.T) { testWalkNotFound(t).WalkR() }
func TestWalkNotFoundMaskError(t *testing.T) {
// this doesn't work for WalkR
newListDirs(t, nil, true,
listResults{
"": {err: ErrorDirNotFound},
},
errorMap{
"": nil,
},
nil,
).Walk()
}
func TestWalkNotFoundSkipkError(t *testing.T) {
// this doesn't work for WalkR
newListDirs(t, nil, true,
listResults{
"": {err: ErrorDirNotFound},
},
errorMap{
"": ErrorSkipDir,
},
nil,
).Walk()
}
func testWalkLevels(t *testing.T, maxLevel int) *listDirs {
da := newDir("a")
oA := mockObject("A")
db := newDir("a/b")
oB := mockObject("a/B")
dc := newDir("a/b/c")
oC := mockObject("a/b/C")
dd := newDir("a/b/c/d")
oD := mockObject("a/b/c/D")
return newListDirs(t, nil, false,
listResults{
"": {entries: DirEntries{oA, da}, err: nil},
"a": {entries: DirEntries{oB, db}, err: nil},
"a/b": {entries: DirEntries{oC, dc}, err: nil},
"a/b/c": {entries: DirEntries{oD, dd}, err: nil},
"a/b/c/d": {entries: DirEntries{}, err: nil},
},
errorMap{
"": nil,
"a": nil,
"a/b": nil,
"a/b/c": nil,
"a/b/c/d": nil,
},
nil,
).SetLevel(maxLevel)
}
func TestWalkLevels(t *testing.T) { testWalkLevels(t, -1).Walk() }
func TestWalkRLevels(t *testing.T) { testWalkLevels(t, -1).WalkR() }
func TestWalkLevelsNoRecursive10(t *testing.T) { testWalkLevels(t, 10).Walk() }
func TestWalkRLevelsNoRecursive10(t *testing.T) { testWalkLevels(t, 10).WalkR() }
func testWalkLevelsNoRecursive(t *testing.T) *listDirs {
da := newDir("a")
oA := mockObject("A")
return newListDirs(t, nil, false,
listResults{
"": {entries: DirEntries{oA, da}, err: nil},
},
errorMap{
"": nil,
},
nil,
).SetLevel(1)
}
func TestWalkLevelsNoRecursive(t *testing.T) { testWalkLevelsNoRecursive(t).Walk() }
func TestWalkRLevelsNoRecursive(t *testing.T) { testWalkLevelsNoRecursive(t).WalkR() }
func testWalkLevels2(t *testing.T) *listDirs {
da := newDir("a")
oA := mockObject("A")
db := newDir("a/b")
oB := mockObject("a/B")
return newListDirs(t, nil, false,
listResults{
"": {entries: DirEntries{oA, da}, err: nil},
"a": {entries: DirEntries{oB, db}, err: nil},
},
errorMap{
"": nil,
"a": nil,
},
nil,
).SetLevel(2)
}
func TestWalkLevels2(t *testing.T) { testWalkLevels2(t).Walk() }
func TestWalkRLevels2(t *testing.T) { testWalkLevels2(t).WalkR() }
func testWalkSkip(t *testing.T) *listDirs {
da := newDir("a")
db := newDir("a/b")
dc := newDir("a/b/c")
return newListDirs(t, nil, false,
listResults{
"": {entries: DirEntries{da}, err: nil},
"a": {entries: DirEntries{db}, err: nil},
"a/b": {entries: DirEntries{dc}, err: nil},
},
errorMap{
"": nil,
"a": nil,
"a/b": ErrorSkipDir,
},
nil,
)
}
func TestWalkSkip(t *testing.T) { testWalkSkip(t).Walk() }
func TestWalkRSkip(t *testing.T) { testWalkSkip(t).WalkR() }
func testWalkErrors(t *testing.T) *listDirs {
lr := listResults{}
em := errorMap{}
de := make(DirEntries, 10)
for i := range de {
path := string('0' + i)
de[i] = newDir(path)
lr[path] = listResult{entries: nil, err: ErrorDirNotFound}
em[path] = ErrorDirNotFound
}
lr[""] = listResult{entries: de, err: nil}
em[""] = nil
return newListDirs(t, nil, true,
lr,
em,
ErrorDirNotFound,
).NoCheckMaps()
}
func TestWalkErrors(t *testing.T) { testWalkErrors(t).Walk() }
func TestWalkRErrors(t *testing.T) { testWalkErrors(t).WalkR() }
var errorBoom = errors.New("boom")
func makeTree(level int, terminalErrors bool) (listResults, errorMap) {
lr := listResults{}
em := errorMap{}
var fill func(path string, level int)
fill = func(path string, level int) {
de := DirEntries{}
if level > 0 {
for _, a := range "0123456789" {
subPath := string(a)
if path != "" {
subPath = path + "/" + subPath
}
de = append(de, newDir(subPath))
fill(subPath, level-1)
}
}
lr[path] = listResult{entries: de, err: nil}
em[path] = nil
if level == 0 && terminalErrors {
em[path] = errorBoom
}
}
fill("", level)
return lr, em
}
func testWalkMulti(t *testing.T) *listDirs {
lr, em := makeTree(3, false)
return newListDirs(t, nil, true,
lr,
em,
nil,
)
}
func TestWalkMulti(t *testing.T) { testWalkMulti(t).Walk() }
func TestWalkRMulti(t *testing.T) { testWalkMulti(t).WalkR() }
func testWalkMultiErrors(t *testing.T) *listDirs {
lr, em := makeTree(3, true)
return newListDirs(t, nil, true,
lr,
em,
errorBoom,
).NoCheckMaps()
}
func TestWalkMultiErrors(t *testing.T) { testWalkMultiErrors(t).Walk() }
func TestWalkRMultiErrors(t *testing.T) { testWalkMultiErrors(t).Walk() }
// a very simple listRcallback function
func makeListRCallback(entries DirEntries, err error) listRFunc {
return func(f Fs, dir string, callback listRCallback) error {
if err == nil {
err = callback(entries)
}
return err
}
}
func TestWalkRDirTree(t *testing.T) {
for _, test := range []struct {
entries DirEntries
want string
err error
root string
level int
}{
{DirEntries{}, "/\n", nil, "", -1},
{DirEntries{mockObject("a")}, `/
a
`, nil, "", -1},
{DirEntries{mockObject("a/b")}, `/
a/
a/
b
`, nil, "", -1},
{DirEntries{mockObject("a/b/c/d")}, `/
a/
a/
b/
a/b/
c/
a/b/c/
d
`, nil, "", -1},
{DirEntries{mockObject("a")}, "", errorBoom, "", -1},
{DirEntries{
mockObject("0/1/2/3"),
mockObject("4/5/6/7"),
mockObject("8/9/a/b"),
mockObject("c/d/e/f"),
mockObject("g/h/i/j"),
mockObject("k/l/m/n"),
mockObject("o/p/q/r"),
mockObject("s/t/u/v"),
mockObject("w/x/y/z"),
}, `/
0/
4/
8/
c/
g/
k/
o/
s/
w/
0/
1/
0/1/
2/
0/1/2/
3
4/
5/
4/5/
6/
4/5/6/
7
8/
9/
8/9/
a/
8/9/a/
b
c/
d/
c/d/
e/
c/d/e/
f
g/
h/
g/h/
i/
g/h/i/
j
k/
l/
k/l/
m/
k/l/m/
n
o/
p/
o/p/
q/
o/p/q/
r
s/
t/
s/t/
u/
s/t/u/
v
w/
x/
w/x/
y/
w/x/y/
z
`, nil, "", -1},
{DirEntries{
mockObject("a/b/c/d/e/f1"),
mockObject("a/b/c/d/e/f2"),
mockObject("a/b/c/d/e/f3"),
}, `a/b/c/
d/
a/b/c/d/
e/
a/b/c/d/e/
f1
f2
f3
`, nil, "a/b/c", -1},
{DirEntries{
mockObject("A"),
mockObject("a/B"),
mockObject("a/b/C"),
mockObject("a/b/c/D"),
mockObject("a/b/c/d/E"),
}, `/
A
a/
a/
B
b/
`, nil, "", 2},
{DirEntries{
mockObject("a/b/c"),
mockObject("a/b/c/d/e"),
}, `/
a/
a/
b/
`, nil, "", 2},
} {
r, err := walkRDirTree(nil, test.root, true, test.level, makeListRCallback(test.entries, test.err))
assert.Equal(t, test.err, err, fmt.Sprintf("%+v", test))
assert.Equal(t, test.want, r.String(), fmt.Sprintf("%+v", test))
}
}