KurlesHS/rclone
1
0
Fork 0
forked from TrueCloudLab/rclone
Code Pull requests Activity

Improve directory listing tests to detect issue #1165

Browse source
This commit is contained in:
Nick Craig-Wood 2017-02-22 10:14:40 +00:00
parent 980cd5bfd8
commit 73a96dc588
2 changed files with 43 additions and 19 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
fstest/fstest.go
View file

@ -27,7 +27,8 @@ import (
var ( var (
// MatchTestRemote matches the remote names used for testing // MatchTestRemote matches the remote names used for testing
MatchTestRemote = regexp.MustCompile(`^rclone-test-[abcdefghijklmnopqrstuvwxyz0123456789]{24}$`) MatchTestRemote = regexp.MustCompile(`^rclone-test-[abcdefghijklmnopqrstuvwxyz0123456789]{24}$`)
listRetries = flag.Int("list-retries", 6, "Number or times to retry listing") // ListRetries is the number of times to retry a listing to overcome eventual consistency
ListRetries = flag.Int("list-retries", 6, "Number or times to retry listing")
) )
// Seed the random number generator // Seed the random number generator
@ -185,7 +186,7 @@ func CheckListingWithPrecision(t *testing.T, f fs.Fs, items []Item, expectedDirs
var objs []fs.Object var objs []fs.Object
var dirs []*fs.Dir var dirs []*fs.Dir
var err error var err error
var retries = *listRetries var retries = *ListRetries
sleep := time.Second / 2 sleep := time.Second / 2
wantListing1, wantListing2 := makeListingFromItems(items) wantListing1, wantListing2 := makeListingFromItems(items)
gotListing := "<unset>" gotListing := "<unset>"

57
fstest/fstests/fstests.go
View file

@ -54,8 +54,6 @@ var (
// ExtraConfigItem describes a config item added on the fly while testing // ExtraConfigItem describes a config item added on the fly while testing
type ExtraConfigItem struct{ Name, Key, Value string } type ExtraConfigItem struct{ Name, Key, Value string }
const eventualConsistencyRetries = 10
func init() { func init() {
flag.StringVar(&RemoteName, "remote", "", "Set this to override the default remote name (eg s3:)") flag.StringVar(&RemoteName, "remote", "", "Set this to override the default remote name (eg s3:)")
} }
@ -205,12 +203,12 @@ func TestFsNewObjectNotFound(t *testing.T) {
func findObject(t *testing.T, Name string) fs.Object { func findObject(t *testing.T, Name string) fs.Object {
var obj fs.Object var obj fs.Object
var err error var err error
for i := 1; i <= eventualConsistencyRetries; i++ { for i := 1; i <= *fstest.ListRetries; i++ {
obj, err = remote.NewObject(Name) obj, err = remote.NewObject(Name)
if err == nil { if err == nil {
break break
} }
t.Logf("Sleeping for 1 second for findObject eventual consistency: %d/%d (%v)", i, eventualConsistencyRetries, err) t.Logf("Sleeping for 1 second for findObject eventual consistency: %d/%d (%v)", i, *fstest.ListRetries, err)
time.Sleep(1 * time.Second) time.Sleep(1 * time.Second)
} }
require.NoError(t, err) require.NoError(t, err)
@ -298,23 +296,48 @@ func TestFsUpdateFile1(t *testing.T) {
// Note that the next test will check there are no duplicated file names // Note that the next test will check there are no duplicated file names
} }
// TestFsListDirFile2 tests the files are correctly uploaded // TestFsListDirFile2 tests the files are correctly uploaded by doing
// Depth 1 directory listings
func TestFsListDirFile2(t *testing.T) { func TestFsListDirFile2(t *testing.T) {
skipIfNotOk(t) skipIfNotOk(t)
var objNames, dirNames []string list := func(dir string, expectedDirNames, expectedObjNames []string) {
for i := 1; i <= eventualConsistencyRetries; i++ { var objNames, dirNames []string
objs, dirs, err := fs.NewLister().SetLevel(1).Start(remote, "").GetAll() for i := 1; i <= *fstest.ListRetries; i++ {
require.NoError(t, err) objs, dirs, err := fs.NewLister().SetLevel(1).Start(remote, dir).GetAll()
objNames = objsToNames(objs) if err == fs.ErrorDirNotFound {
dirNames = dirsToNames(dirs) objs, dirs, err = fs.NewLister().SetLevel(1).Start(remote, winPath(dir)).GetAll()
if len(objNames) >= 1 && len(dirNames) >= 1 { }
break require.NoError(t, err)
objNames = objsToNames(objs)
dirNames = dirsToNames(dirs)
if len(objNames) >= len(expectedObjNames) && len(dirNames) >= len(expectedDirNames) {
break
}
t.Logf("Sleeping for 1 second for TestFsListDirFile2 eventual consistency: %d/%d", i, *fstest.ListRetries)
time.Sleep(1 * time.Second)
} }
t.Logf("Sleeping for 1 second for TestFsListDirFile2 eventual consistency: %d/%d", i, eventualConsistencyRetries) assert.Equal(t, expectedDirNames, dirNames)
time.Sleep(1 * time.Second) assert.Equal(t, expectedObjNames, objNames)
}
dir := file2.Path
deepest := true
for dir != "" {
expectedObjNames := []string{}
expectedDirNames := []string{}
child := dir
dir = path.Dir(dir)
if dir == "." {
dir = ""
expectedObjNames = append(expectedObjNames, winPath(file1.Path))
}
if deepest {
expectedObjNames = append(expectedObjNames, winPath(file2.Path))
deepest = false
} else {
expectedDirNames = append(expectedDirNames, winPath(child))
}
list(dir, expectedDirNames, expectedObjNames)
} }
assert.Equal(t, []string{`hello_ sausage`}, dirNames)
assert.Equal(t, []string{file1.Path}, objNames)
} }
// TestFsListDirRoot tests that DirList works in the root // TestFsListDirRoot tests that DirList works in the root