8a6a8b9623
This simplifies the implementation of remotes. The only required interface is now `List` which is a simple one level directory list. Optionally remotes may implement `ListR` if they have an efficient way of doing a recursive list.
563 lines
12 KiB
Go
563 lines
12 KiB
Go
package fs
|
|
|
|
import (
|
|
"fmt"
|
|
"io"
|
|
"sync"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/pkg/errors"
|
|
"github.com/stretchr/testify/assert"
|
|
"github.com/stretchr/testify/require"
|
|
)
|
|
|
|
type (
|
|
listResult struct {
|
|
entries DirEntries
|
|
err error
|
|
}
|
|
|
|
listResults map[string]listResult
|
|
|
|
errorMap map[string]error
|
|
|
|
listDirs struct {
|
|
mu sync.Mutex
|
|
t *testing.T
|
|
fs Fs
|
|
includeAll bool
|
|
results listResults
|
|
walkResults listResults
|
|
walkErrors errorMap
|
|
finalError error
|
|
checkMaps bool
|
|
maxLevel int
|
|
}
|
|
)
|
|
|
|
var errNotImpl = errors.New("not implemented")
|
|
|
|
type mockObject string
|
|
|
|
func (o mockObject) String() string { return string(o) }
|
|
func (o mockObject) Fs() Info { return nil }
|
|
func (o mockObject) Remote() string { return string(o) }
|
|
func (o mockObject) Hash(HashType) (string, error) { return "", errNotImpl }
|
|
func (o mockObject) ModTime() (t time.Time) { return t }
|
|
func (o mockObject) Size() int64 { return 0 }
|
|
func (o mockObject) Storable() bool { return true }
|
|
func (o mockObject) SetModTime(time.Time) error { return errNotImpl }
|
|
func (o mockObject) Open(options ...OpenOption) (io.ReadCloser, error) { return nil, errNotImpl }
|
|
func (o mockObject) Update(in io.Reader, src ObjectInfo, options ...OpenOption) error {
|
|
return errNotImpl
|
|
}
|
|
func (o mockObject) Remove() error { return errNotImpl }
|
|
|
|
func newListDirs(t *testing.T, f Fs, includeAll bool, results listResults, walkErrors errorMap, finalError error) *listDirs {
|
|
return &listDirs{
|
|
t: t,
|
|
fs: f,
|
|
includeAll: includeAll,
|
|
results: results,
|
|
walkErrors: walkErrors,
|
|
walkResults: listResults{},
|
|
finalError: finalError,
|
|
checkMaps: true,
|
|
maxLevel: -1,
|
|
}
|
|
}
|
|
|
|
// NoCheckMaps marks the maps as to be ignored at the end
|
|
func (ls *listDirs) NoCheckMaps() *listDirs {
|
|
ls.checkMaps = false
|
|
return ls
|
|
}
|
|
|
|
// SetLevel(1) turns off recursion
|
|
func (ls *listDirs) SetLevel(maxLevel int) *listDirs {
|
|
ls.maxLevel = maxLevel
|
|
return ls
|
|
}
|
|
|
|
// ListDir returns the expected listing for the directory
|
|
func (ls *listDirs) ListDir(f Fs, includeAll bool, dir string) (entries DirEntries, err error) {
|
|
ls.mu.Lock()
|
|
defer ls.mu.Unlock()
|
|
assert.Equal(ls.t, ls.fs, f)
|
|
assert.Equal(ls.t, ls.includeAll, includeAll)
|
|
|
|
// Fetch results for this path
|
|
result, ok := ls.results[dir]
|
|
if !ok {
|
|
ls.t.Errorf("Unexpected list of %q", dir)
|
|
return nil, errors.New("unexpected list")
|
|
}
|
|
delete(ls.results, dir)
|
|
|
|
// Put expected results for call of WalkFn
|
|
ls.walkResults[dir] = result
|
|
|
|
return result.entries, result.err
|
|
}
|
|
|
|
// ListR returns the expected listing for the directory using ListR
|
|
func (ls *listDirs) ListR(dir string, callback ListRCallback) (err error) {
|
|
ls.mu.Lock()
|
|
defer ls.mu.Unlock()
|
|
|
|
var errorReturn error
|
|
for dirPath, result := range ls.results {
|
|
// Put expected results for call of WalkFn
|
|
// Note that we don't call the function at all if we got an error
|
|
if result.err != nil {
|
|
errorReturn = result.err
|
|
}
|
|
if errorReturn == nil {
|
|
err = callback(result.entries)
|
|
require.NoError(ls.t, err)
|
|
ls.walkResults[dirPath] = result
|
|
}
|
|
}
|
|
ls.results = listResults{}
|
|
return errorReturn
|
|
}
|
|
|
|
// IsFinished checks everything expected was used up
|
|
func (ls *listDirs) IsFinished() {
|
|
if ls.checkMaps {
|
|
assert.Equal(ls.t, errorMap{}, ls.walkErrors)
|
|
assert.Equal(ls.t, listResults{}, ls.results)
|
|
assert.Equal(ls.t, listResults{}, ls.walkResults)
|
|
}
|
|
}
|
|
|
|
// WalkFn is called by the walk to test the expectations
|
|
func (ls *listDirs) WalkFn(dir string, entries DirEntries, err error) error {
|
|
ls.mu.Lock()
|
|
defer ls.mu.Unlock()
|
|
// ls.t.Logf("WalkFn(%q, %v, %q)", dir, entries, err)
|
|
|
|
// Fetch expected entries and err
|
|
result, ok := ls.walkResults[dir]
|
|
if !ok {
|
|
ls.t.Errorf("Unexpected walk of %q (result not found)", dir)
|
|
return errors.New("result not found")
|
|
}
|
|
delete(ls.walkResults, dir)
|
|
|
|
// Check arguments are as expected
|
|
assert.Equal(ls.t, result.entries, entries)
|
|
assert.Equal(ls.t, result.err, err)
|
|
|
|
// Fetch return value
|
|
returnErr, ok := ls.walkErrors[dir]
|
|
if !ok {
|
|
ls.t.Errorf("Unexpected walk of %q (error not found)", dir)
|
|
return errors.New("error not found")
|
|
}
|
|
delete(ls.walkErrors, dir)
|
|
|
|
return returnErr
|
|
}
|
|
|
|
// Walk does the walk and tests the expectations
|
|
func (ls *listDirs) Walk() {
|
|
err := walk(nil, "", ls.includeAll, ls.maxLevel, ls.WalkFn, ls.ListDir)
|
|
assert.Equal(ls.t, ls.finalError, err)
|
|
ls.IsFinished()
|
|
}
|
|
|
|
// WalkR does the walkR and tests the expectations
|
|
func (ls *listDirs) WalkR() {
|
|
err := walkR(nil, "", ls.includeAll, ls.maxLevel, ls.WalkFn, ls.ListR)
|
|
assert.Equal(ls.t, ls.finalError, err)
|
|
if ls.finalError == nil {
|
|
ls.IsFinished()
|
|
}
|
|
}
|
|
|
|
func newDir(name string) *Dir {
|
|
return &Dir{Name: name}
|
|
}
|
|
|
|
func testWalkEmpty(t *testing.T) *listDirs {
|
|
return newListDirs(t, nil, false,
|
|
listResults{
|
|
"": {entries: DirEntries{}, err: nil},
|
|
},
|
|
errorMap{
|
|
"": nil,
|
|
},
|
|
nil,
|
|
)
|
|
}
|
|
func TestWalkEmpty(t *testing.T) { testWalkEmpty(t).Walk() }
|
|
func TestWalkREmpty(t *testing.T) { testWalkEmpty(t).WalkR() }
|
|
|
|
func testWalkEmptySkip(t *testing.T) *listDirs {
|
|
return newListDirs(t, nil, true,
|
|
listResults{
|
|
"": {entries: DirEntries{}, err: nil},
|
|
},
|
|
errorMap{
|
|
"": ErrorSkipDir,
|
|
},
|
|
nil,
|
|
)
|
|
}
|
|
func TestWalkEmptySkip(t *testing.T) { testWalkEmptySkip(t).Walk() }
|
|
func TestWalkREmptySkip(t *testing.T) { testWalkEmptySkip(t).WalkR() }
|
|
|
|
func testWalkNotFound(t *testing.T) *listDirs {
|
|
return newListDirs(t, nil, true,
|
|
listResults{
|
|
"": {err: ErrorDirNotFound},
|
|
},
|
|
errorMap{
|
|
"": ErrorDirNotFound,
|
|
},
|
|
ErrorDirNotFound,
|
|
)
|
|
}
|
|
func TestWalkNotFound(t *testing.T) { testWalkNotFound(t).Walk() }
|
|
func TestWalkRNotFound(t *testing.T) { testWalkNotFound(t).WalkR() }
|
|
|
|
func TestWalkNotFoundMaskError(t *testing.T) {
|
|
// this doesn't work for WalkR
|
|
newListDirs(t, nil, true,
|
|
listResults{
|
|
"": {err: ErrorDirNotFound},
|
|
},
|
|
errorMap{
|
|
"": nil,
|
|
},
|
|
nil,
|
|
).Walk()
|
|
}
|
|
|
|
func TestWalkNotFoundSkipkError(t *testing.T) {
|
|
// this doesn't work for WalkR
|
|
newListDirs(t, nil, true,
|
|
listResults{
|
|
"": {err: ErrorDirNotFound},
|
|
},
|
|
errorMap{
|
|
"": ErrorSkipDir,
|
|
},
|
|
nil,
|
|
).Walk()
|
|
}
|
|
|
|
func testWalkLevels(t *testing.T, maxLevel int) *listDirs {
|
|
da := newDir("a")
|
|
oA := mockObject("A")
|
|
db := newDir("a/b")
|
|
oB := mockObject("a/B")
|
|
dc := newDir("a/b/c")
|
|
oC := mockObject("a/b/C")
|
|
dd := newDir("a/b/c/d")
|
|
oD := mockObject("a/b/c/D")
|
|
return newListDirs(t, nil, false,
|
|
listResults{
|
|
"": {entries: DirEntries{oA, da}, err: nil},
|
|
"a": {entries: DirEntries{oB, db}, err: nil},
|
|
"a/b": {entries: DirEntries{oC, dc}, err: nil},
|
|
"a/b/c": {entries: DirEntries{oD, dd}, err: nil},
|
|
"a/b/c/d": {entries: DirEntries{}, err: nil},
|
|
},
|
|
errorMap{
|
|
"": nil,
|
|
"a": nil,
|
|
"a/b": nil,
|
|
"a/b/c": nil,
|
|
"a/b/c/d": nil,
|
|
},
|
|
nil,
|
|
).SetLevel(maxLevel)
|
|
}
|
|
func TestWalkLevels(t *testing.T) { testWalkLevels(t, -1).Walk() }
|
|
func TestWalkRLevels(t *testing.T) { testWalkLevels(t, -1).WalkR() }
|
|
func TestWalkLevelsNoRecursive10(t *testing.T) { testWalkLevels(t, 10).Walk() }
|
|
func TestWalkRLevelsNoRecursive10(t *testing.T) { testWalkLevels(t, 10).WalkR() }
|
|
|
|
func testWalkLevelsNoRecursive(t *testing.T) *listDirs {
|
|
da := newDir("a")
|
|
oA := mockObject("A")
|
|
return newListDirs(t, nil, false,
|
|
listResults{
|
|
"": {entries: DirEntries{oA, da}, err: nil},
|
|
},
|
|
errorMap{
|
|
"": nil,
|
|
},
|
|
nil,
|
|
).SetLevel(1)
|
|
}
|
|
func TestWalkLevelsNoRecursive(t *testing.T) { testWalkLevelsNoRecursive(t).Walk() }
|
|
func TestWalkRLevelsNoRecursive(t *testing.T) { testWalkLevelsNoRecursive(t).WalkR() }
|
|
|
|
func testWalkLevels2(t *testing.T) *listDirs {
|
|
da := newDir("a")
|
|
oA := mockObject("A")
|
|
db := newDir("a/b")
|
|
oB := mockObject("a/B")
|
|
return newListDirs(t, nil, false,
|
|
listResults{
|
|
"": {entries: DirEntries{oA, da}, err: nil},
|
|
"a": {entries: DirEntries{oB, db}, err: nil},
|
|
},
|
|
errorMap{
|
|
"": nil,
|
|
"a": nil,
|
|
},
|
|
nil,
|
|
).SetLevel(2)
|
|
}
|
|
func TestWalkLevels2(t *testing.T) { testWalkLevels2(t).Walk() }
|
|
func TestWalkRLevels2(t *testing.T) { testWalkLevels2(t).WalkR() }
|
|
|
|
func testWalkSkip(t *testing.T) *listDirs {
|
|
da := newDir("a")
|
|
db := newDir("a/b")
|
|
dc := newDir("a/b/c")
|
|
return newListDirs(t, nil, false,
|
|
listResults{
|
|
"": {entries: DirEntries{da}, err: nil},
|
|
"a": {entries: DirEntries{db}, err: nil},
|
|
"a/b": {entries: DirEntries{dc}, err: nil},
|
|
},
|
|
errorMap{
|
|
"": nil,
|
|
"a": nil,
|
|
"a/b": ErrorSkipDir,
|
|
},
|
|
nil,
|
|
)
|
|
}
|
|
func TestWalkSkip(t *testing.T) { testWalkSkip(t).Walk() }
|
|
func TestWalkRSkip(t *testing.T) { testWalkSkip(t).WalkR() }
|
|
|
|
func testWalkErrors(t *testing.T) *listDirs {
|
|
lr := listResults{}
|
|
em := errorMap{}
|
|
de := make(DirEntries, 10)
|
|
for i := range de {
|
|
path := string('0' + i)
|
|
de[i] = newDir(path)
|
|
lr[path] = listResult{entries: nil, err: ErrorDirNotFound}
|
|
em[path] = ErrorDirNotFound
|
|
}
|
|
lr[""] = listResult{entries: de, err: nil}
|
|
em[""] = nil
|
|
return newListDirs(t, nil, true,
|
|
lr,
|
|
em,
|
|
ErrorDirNotFound,
|
|
).NoCheckMaps()
|
|
}
|
|
func TestWalkErrors(t *testing.T) { testWalkErrors(t).Walk() }
|
|
func TestWalkRErrors(t *testing.T) { testWalkErrors(t).WalkR() }
|
|
|
|
var errorBoom = errors.New("boom")
|
|
|
|
func makeTree(level int, terminalErrors bool) (listResults, errorMap) {
|
|
lr := listResults{}
|
|
em := errorMap{}
|
|
var fill func(path string, level int)
|
|
fill = func(path string, level int) {
|
|
de := DirEntries{}
|
|
if level > 0 {
|
|
for _, a := range "0123456789" {
|
|
subPath := string(a)
|
|
if path != "" {
|
|
subPath = path + "/" + subPath
|
|
}
|
|
de = append(de, newDir(subPath))
|
|
fill(subPath, level-1)
|
|
}
|
|
}
|
|
lr[path] = listResult{entries: de, err: nil}
|
|
em[path] = nil
|
|
if level == 0 && terminalErrors {
|
|
em[path] = errorBoom
|
|
}
|
|
}
|
|
fill("", level)
|
|
return lr, em
|
|
}
|
|
|
|
func testWalkMulti(t *testing.T) *listDirs {
|
|
lr, em := makeTree(3, false)
|
|
return newListDirs(t, nil, true,
|
|
lr,
|
|
em,
|
|
nil,
|
|
)
|
|
}
|
|
func TestWalkMulti(t *testing.T) { testWalkMulti(t).Walk() }
|
|
func TestWalkRMulti(t *testing.T) { testWalkMulti(t).WalkR() }
|
|
|
|
func testWalkMultiErrors(t *testing.T) *listDirs {
|
|
lr, em := makeTree(3, true)
|
|
return newListDirs(t, nil, true,
|
|
lr,
|
|
em,
|
|
errorBoom,
|
|
).NoCheckMaps()
|
|
}
|
|
func TestWalkMultiErrors(t *testing.T) { testWalkMultiErrors(t).Walk() }
|
|
func TestWalkRMultiErrors(t *testing.T) { testWalkMultiErrors(t).Walk() }
|
|
|
|
// a very simple listRcallback function
|
|
func makeListRCallback(entries DirEntries, err error) ListRFn {
|
|
return func(dir string, callback ListRCallback) error {
|
|
if err == nil {
|
|
err = callback(entries)
|
|
}
|
|
return err
|
|
}
|
|
}
|
|
|
|
func TestWalkRDirTree(t *testing.T) {
|
|
for _, test := range []struct {
|
|
entries DirEntries
|
|
want string
|
|
err error
|
|
root string
|
|
level int
|
|
}{
|
|
{DirEntries{}, "/\n", nil, "", -1},
|
|
{DirEntries{mockObject("a")}, `/
|
|
a
|
|
`, nil, "", -1},
|
|
{DirEntries{mockObject("a/b")}, `/
|
|
a/
|
|
a/
|
|
b
|
|
`, nil, "", -1},
|
|
{DirEntries{mockObject("a/b/c/d")}, `/
|
|
a/
|
|
a/
|
|
b/
|
|
a/b/
|
|
c/
|
|
a/b/c/
|
|
d
|
|
`, nil, "", -1},
|
|
{DirEntries{mockObject("a")}, "", errorBoom, "", -1},
|
|
{DirEntries{
|
|
mockObject("0/1/2/3"),
|
|
mockObject("4/5/6/7"),
|
|
mockObject("8/9/a/b"),
|
|
mockObject("c/d/e/f"),
|
|
mockObject("g/h/i/j"),
|
|
mockObject("k/l/m/n"),
|
|
mockObject("o/p/q/r"),
|
|
mockObject("s/t/u/v"),
|
|
mockObject("w/x/y/z"),
|
|
}, `/
|
|
0/
|
|
4/
|
|
8/
|
|
c/
|
|
g/
|
|
k/
|
|
o/
|
|
s/
|
|
w/
|
|
0/
|
|
1/
|
|
0/1/
|
|
2/
|
|
0/1/2/
|
|
3
|
|
4/
|
|
5/
|
|
4/5/
|
|
6/
|
|
4/5/6/
|
|
7
|
|
8/
|
|
9/
|
|
8/9/
|
|
a/
|
|
8/9/a/
|
|
b
|
|
c/
|
|
d/
|
|
c/d/
|
|
e/
|
|
c/d/e/
|
|
f
|
|
g/
|
|
h/
|
|
g/h/
|
|
i/
|
|
g/h/i/
|
|
j
|
|
k/
|
|
l/
|
|
k/l/
|
|
m/
|
|
k/l/m/
|
|
n
|
|
o/
|
|
p/
|
|
o/p/
|
|
q/
|
|
o/p/q/
|
|
r
|
|
s/
|
|
t/
|
|
s/t/
|
|
u/
|
|
s/t/u/
|
|
v
|
|
w/
|
|
x/
|
|
w/x/
|
|
y/
|
|
w/x/y/
|
|
z
|
|
`, nil, "", -1},
|
|
{DirEntries{
|
|
mockObject("a/b/c/d/e/f1"),
|
|
mockObject("a/b/c/d/e/f2"),
|
|
mockObject("a/b/c/d/e/f3"),
|
|
}, `a/b/c/
|
|
d/
|
|
a/b/c/d/
|
|
e/
|
|
a/b/c/d/e/
|
|
f1
|
|
f2
|
|
f3
|
|
`, nil, "a/b/c", -1},
|
|
{DirEntries{
|
|
mockObject("A"),
|
|
mockObject("a/B"),
|
|
mockObject("a/b/C"),
|
|
mockObject("a/b/c/D"),
|
|
mockObject("a/b/c/d/E"),
|
|
}, `/
|
|
A
|
|
a/
|
|
a/
|
|
B
|
|
b/
|
|
`, nil, "", 2},
|
|
{DirEntries{
|
|
mockObject("a/b/c"),
|
|
mockObject("a/b/c/d/e"),
|
|
}, `/
|
|
a/
|
|
a/
|
|
b/
|
|
`, nil, "", 2},
|
|
} {
|
|
r, err := walkRDirTree(nil, test.root, true, test.level, makeListRCallback(test.entries, test.err))
|
|
assert.Equal(t, test.err, err, fmt.Sprintf("%+v", test))
|
|
assert.Equal(t, test.want, r.String(), fmt.Sprintf("%+v", test))
|
|
}
|
|
}
|