TrueCloudLab/rclone
16
0
Fork 2
Code Issues Pull requests 1 Projects Releases Packages Wiki Activity Actions

vfstests: move functional tests from mountlib and make them work with VFS

Browse source 
The tests are now run for the mount commands and for the plain VFS.

This makes the tests much easier to debug when running with a VFS than
through a mount.
This commit is contained in:
Nick Craig-Wood 2020-04-16 13:33:46 +01:00
parent b25f5eb0d1
commit fd39cbc193
16 changed files with 259 additions and 67 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
cmd/cmount/mount_test.go
View file

@ -11,9 +11,9 @@ package cmount
import (
"testing"
"github.com/rclone/rclone/cmd/mountlib/mounttest"
"github.com/rclone/rclone/vfs/vfstest"
)
func TestMount(t *testing.T) {
mounttest.RunTests(t, mount)
vfstest.RunTests(t, false, mount)
}

4
cmd/mount/mount_test.go
View file

@ -6,12 +6,12 @@ import (
"runtime"
"testing"
"github.com/rclone/rclone/cmd/mountlib/mounttest"
"github.com/rclone/rclone/vfs/vfstest"
)
func TestMount(t *testing.T) {
if runtime.NumCPU() <= 2 {
t.Skip("FIXME skipping mount tests as they lock up on <= 2 CPUs - See: https://github.com/rclone/rclone/issues/3154")
}
mounttest.RunTests(t, mount)
vfstest.RunTests(t, false, mount)
}

4
cmd/mount2/mount_test.go
View file

@ -5,9 +5,9 @@ package mount2
import (
"testing"
"github.com/rclone/rclone/cmd/mountlib/mounttest"
"github.com/rclone/rclone/vfs/vfstest"
)
func TestMount(t *testing.T) {
mounttest.RunTests(t, mount)
vfstest.RunTests(t, false, mount)
}

25
cmd/mountlib/mounttest/dir.go → vfs/vfstest/dir.go
View file

@ -1,8 +1,7 @@
package mounttest
package vfstest
import (
"context"
"os"
"testing"
"time"
@ -37,7 +36,7 @@ func TestDirCreateAndRemoveDir(t *testing.T) {
run.checkDir(t, "dir/|dir/subdir/")
// Check we can't delete a directory with stuff in
err := os.Remove(run.path("dir"))
err := run.os.Remove(run.path("dir"))
assert.Error(t, err, "file exists")
// Now delete subdir then dir - should produce no errors
@ -56,7 +55,7 @@ func TestDirCreateAndRemoveFile(t *testing.T) {
run.checkDir(t, "dir/|dir/file 6")
// Check we can't delete a directory with stuff in
err := os.Remove(run.path("dir"))
err := run.os.Remove(run.path("dir"))
assert.Error(t, err, "file exists")
// Now delete file
@ -75,14 +74,14 @@ func TestDirRenameFile(t *testing.T) {
run.createFile(t, "file", "potato")
run.checkDir(t, "dir/|file 6")
err := os.Rename(run.path("file"), run.path("file2"))
err := run.os.Rename(run.path("file"), run.path("file2"))
require.NoError(t, err)
run.checkDir(t, "dir/|file2 6")
data := run.readFile(t, "file2")
assert.Equal(t, "potato", data)
err = os.Rename(run.path("file2"), run.path("dir/file3"))
err = run.os.Rename(run.path("file2"), run.path("dir/file3"))
require.NoError(t, err)
run.checkDir(t, "dir/|dir/file3 6")
@ -103,11 +102,11 @@ func TestDirRenameEmptyDir(t *testing.T) {
run.mkdir(t, "dir1")
run.checkDir(t, "dir/|dir1/")
err := os.Rename(run.path("dir1"), run.path("dir/dir2"))
err := run.os.Rename(run.path("dir1"), run.path("dir/dir2"))
require.NoError(t, err)
run.checkDir(t, "dir/|dir/dir2/")
err = os.Rename(run.path("dir/dir2"), run.path("dir/dir3"))
err = run.os.Rename(run.path("dir/dir2"), run.path("dir/dir3"))
require.NoError(t, err)
run.checkDir(t, "dir/|dir/dir3/")
@ -125,11 +124,11 @@ func TestDirRenameFullDir(t *testing.T) {
run.createFile(t, "dir1/potato.txt", "maris piper")
run.checkDir(t, "dir/|dir1/|dir1/potato.txt 11")
err := os.Rename(run.path("dir1"), run.path("dir/dir2"))
err := run.os.Rename(run.path("dir1"), run.path("dir/dir2"))
require.NoError(t, err)
run.checkDir(t, "dir/|dir/dir2/|dir/dir2/potato.txt 11")
err = os.Rename(run.path("dir/dir2"), run.path("dir/dir3"))
err = run.os.Rename(run.path("dir/dir2"), run.path("dir/dir3"))
require.NoError(t, err)
run.checkDir(t, "dir/|dir/dir3/|dir/dir3/potato.txt 11")
@ -145,10 +144,10 @@ func TestDirModTime(t *testing.T) {
run.mkdir(t, "dir")
mtime := time.Date(2012, time.November, 18, 17, 32, 31, 0, time.UTC)
err := os.Chtimes(run.path("dir"), mtime, mtime)
err := run.os.Chtimes(run.path("dir"), mtime, mtime)
require.NoError(t, err)
info, err := os.Stat(run.path("dir"))
info, err := run.os.Stat(run.path("dir"))
require.NoError(t, err)
// avoid errors because of timezone differences
@ -214,7 +213,7 @@ func TestDirCacheFlushOnDirRename(t *testing.T) {
run.readLocal(t, localDm, "")
assert.Equal(t, dm, localDm, "expected vs fuse mount")
err = os.Rename(run.path("dir"), run.path("rid"))
err = run.os.Rename(run.path("dir"), run.path("rid"))
require.NoError(t, err)
dm = newDirMap("rid/|rid/subdir/|rid/file 1")

5
cmd/mountlib/mounttest/edge_cases.go → vfs/vfstest/edge_cases.go
View file

@ -1,7 +1,6 @@
package mounttest
package vfstest
import (
"os"
"runtime"
"testing"
@ -43,7 +42,7 @@ func TestRenameOpenHandle(t *testing.T) {
require.NoError(t, err)
// attempt to rename open file
err = os.Rename(path, path+"bla")
err = run.os.Rename(path, path+"bla")
require.NoError(t, err)
// close open writers to allow rename on remote to go through

23
cmd/mountlib/mounttest/file.go → vfs/vfstest/file.go
View file

@ -1,4 +1,4 @@
package mounttest
package vfstest
import (
"os"
@ -6,6 +6,7 @@ import (
"testing"
"time"
"github.com/rclone/rclone/vfs"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
@ -17,10 +18,10 @@ func TestFileModTime(t *testing.T) {
run.createFile(t, "file", "123")
mtime := time.Date(2012, time.November, 18, 17, 32, 31, 0, time.UTC)
err := os.Chtimes(run.path("file"), mtime, mtime)
err := run.os.Chtimes(run.path("file"), mtime, mtime)
require.NoError(t, err)
info, err := os.Stat(run.path("file"))
info, err := run.os.Stat(run.path("file"))
require.NoError(t, err)
// avoid errors because of timezone differences
@ -29,14 +30,14 @@ func TestFileModTime(t *testing.T) {
run.rm(t, "file")
}
// os.Create without opening for write too
func osCreate(name string) (*os.File, error) {
return os.OpenFile(name, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0666)
// run.os.Create without opening for write too
func osCreate(name string) (vfs.OsFiler, error) {
return run.os.OpenFile(name, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0666)
}
// os.Create with append
func osAppend(name string) (*os.File, error) {
return os.OpenFile(name, os.O_WRONLY|os.O_APPEND, 0666)
// run.os.Create with append
func osAppend(name string) (vfs.OsFiler, error) {
return run.os.OpenFile(name, os.O_WRONLY|os.O_APPEND, 0666)
}
// TestFileModTimeWithOpenWriters tests mod time on open files
@ -55,7 +56,7 @@ func TestFileModTimeWithOpenWriters(t *testing.T) {
_, err = f.Write([]byte{104, 105})
require.NoError(t, err)
err = os.Chtimes(filepath, mtime, mtime)
err = run.os.Chtimes(filepath, mtime, mtime)
require.NoError(t, err)
err = f.Close()
@ -63,7 +64,7 @@ func TestFileModTimeWithOpenWriters(t *testing.T) {
run.waitForWriters()
info, err := os.Stat(filepath)
info, err := run.os.Stat(filepath)
require.NoError(t, err)
// avoid errors because of timezone differences

66
cmd/mountlib/mounttest/fs.go → vfs/vfstest/fs.go
View file

@ -1,6 +1,6 @@
// Test suite for rclonefs
package mounttest
package vfstest
import (
"context"
@ -23,7 +23,6 @@ import (
"github.com/rclone/rclone/fs"
"github.com/rclone/rclone/fs/walk"
"github.com/rclone/rclone/fstest"
"github.com/rclone/rclone/lib/file"
"github.com/rclone/rclone/vfs"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
@ -33,7 +32,7 @@ type (
// UnmountFn is called to unmount the file system
UnmountFn func() error
// MountFn is called to mount the file system
MountFn func(f fs.Fs, mountpoint string) (*vfs.VFS, <-chan error, func() error, error)
MountFn func(f fs.Fs, mountpoint string) (vfs *vfs.VFS, unmountResult <-chan error, unmount func() error, err error)
)
var (
@ -41,7 +40,9 @@ var (
)
// RunTests runs all the tests against all the VFS cache modes
func RunTests(t *testing.T, fn MountFn) {
//
// If useVFS is set then it runs the tests against a VFS rather than amount
func RunTests(t *testing.T, useVFS bool, fn MountFn) {
mountFn = fn
flag.Parse()
cacheModes := []vfs.CacheMode{
@ -50,7 +51,7 @@ func RunTests(t *testing.T, fn MountFn) {
vfs.CacheModeWrites,
vfs.CacheModeFull,
}
run = newRun()
run = newRun(useVFS)
for _, cacheMode := range cacheModes {
run.cacheMode(cacheMode)
log.Printf("Starting test run with cache mode %v", cacheMode)
@ -92,7 +93,9 @@ func RunTests(t *testing.T, fn MountFn) {
// Run holds the remotes for a test run
type Run struct {
os Oser
vfs *vfs.VFS
useVFS bool // set if we are testing a VFS not a mount
mountPath string
fremote fs.Fs
fremoteName string
@ -111,11 +114,11 @@ var run *Run
// r.fremote is an empty remote Fs
//
// Finalise() will tidy them away when done.
func newRun() *Run {
func newRun(useVFS bool) *Run {
r := &Run{
useVFS: useVFS,
umountResult: make(chan error, 1),
}
fstest.Initialise()
var err error
@ -129,7 +132,9 @@ func newRun() *Run {
log.Fatalf("Failed to open mkdir %q: %v", *fstest.RemoteName, err)
}
r.mountPath = findMountPath()
if !r.useVFS {
r.mountPath = findMountPath()
}
// Mount it up
r.mount()
@ -169,6 +174,12 @@ func (r *Run) mount() {
} else {
log.Printf("mount OK")
}
if r.useVFS {
r.os = vfsOs{r.vfs}
} else {
r.os = realOs{}
}
}
func (r *Run) umount() {
@ -238,18 +249,31 @@ func (r *Run) skipIfNoFUSE(t *testing.T) {
}
}
func (r *Run) skipIfVFS(t *testing.T) {
if r.useVFS {
t.Skip("Not running under VFS")
}
}
// Finalise cleans the remote and unmounts
func (r *Run) Finalise() {
r.umount()
r.cleanRemote()
err := os.RemoveAll(r.mountPath)
if err != nil {
log.Printf("Failed to clean mountPath %q: %v", r.mountPath, err)
if r.useVFS {
// FIXME
} else {
err := os.RemoveAll(r.mountPath)
if err != nil {
log.Printf("Failed to clean mountPath %q: %v", r.mountPath, err)
}
}
}
// path returns an OS local path for filepath
func (r *Run) path(filePath string) string {
if r.useVFS {
return filePath
}
// return windows drive letter root as E:\
if filePath == "" && runtime.GOOS == "windows" {
return run.mountPath + `\`
@ -284,7 +308,7 @@ func (dm dirMap) filesOnly() dirMap {
// reads the local tree into dir
func (r *Run) readLocal(t *testing.T, dir dirMap, filePath string) {
realPath := r.path(filePath)
files, err := ioutil.ReadDir(realPath)
files, err := r.os.ReadDir(realPath)
require.NoError(t, err)
for _, fi := range files {
name := path.Join(filePath, fi.Name())
@ -353,13 +377,13 @@ func (r *Run) waitForWriters() {
// If there is an error writing then writeFile
// deletes it an existing file and tries again.
func writeFile(filename string, data []byte, perm os.FileMode) error {
f, err := file.OpenFile(filename, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, perm)
f, err := run.os.OpenFile(filename, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, perm)
if err != nil {
err = os.Remove(filename)
err = run.os.Remove(filename)
if err != nil {
return err
}
f, err = file.OpenFile(filename, os.O_WRONLY|os.O_CREATE, perm)
f, err = run.os.OpenFile(filename, os.O_WRONLY|os.O_CREATE, perm)
if err != nil {
return err
}
@ -383,7 +407,7 @@ func (r *Run) createFile(t *testing.T, filepath string, contents string) {
func (r *Run) readFile(t *testing.T, filepath string) string {
filepath = r.path(filepath)
result, err := ioutil.ReadFile(filepath)
result, err := run.os.ReadFile(filepath)
require.NoError(t, err)
time.Sleep(100 * time.Millisecond) // FIXME wait for Release
return string(result)
@ -391,18 +415,18 @@ func (r *Run) readFile(t *testing.T, filepath string) string {
func (r *Run) mkdir(t *testing.T, filepath string) {
filepath = r.path(filepath)
err := os.Mkdir(filepath, 0700)
err := run.os.Mkdir(filepath, 0700)
require.NoError(t, err)
}
func (r *Run) rm(t *testing.T, filepath string) {
filepath = r.path(filepath)
err := os.Remove(filepath)
err := run.os.Remove(filepath)
require.NoError(t, err)
// Wait for file to disappear from listing
for i := 0; i < 100; i++ {
_, err := os.Stat(filepath)
_, err := run.os.Stat(filepath)
if os.IsNotExist(err) {
return
}
@ -413,13 +437,14 @@ func (r *Run) rm(t *testing.T, filepath string) {
func (r *Run) rmdir(t *testing.T, filepath string) {
filepath = r.path(filepath)
err := os.Remove(filepath)
err := run.os.Remove(filepath)
require.NoError(t, err)
}
// TestMount checks that the Fs is mounted by seeing if the mountpoint
// is in the mount output
func TestMount(t *testing.T) {
run.skipIfVFS(t)
run.skipIfNoFUSE(t)
if runtime.GOOS == "windows" {
t.Skip("not running on windows")
@ -432,6 +457,7 @@ func TestMount(t *testing.T) {
// TestRoot checks root directory is present and correct
func TestRoot(t *testing.T) {
run.skipIfVFS(t)
run.skipIfNoFUSE(t)
fi, err := os.Lstat(run.mountPath)

116
vfs/vfstest/os.go Normal file
View file

@ -0,0 +1,116 @@
package vfstest
import (
"io/ioutil"
"os"
"time"
"github.com/rclone/rclone/lib/file"
"github.com/rclone/rclone/vfs"
)
// Oser defines the things that the "os" package can do
//
// This covers what the VFS can do also
type Oser interface {
Chtimes(name string, atime time.Time, mtime time.Time) error
Create(name string) (vfs.Handle, error)
Mkdir(name string, perm os.FileMode) error
Open(name string) (vfs.Handle, error)
OpenFile(name string, flags int, perm os.FileMode) (fd vfs.Handle, err error)
ReadDir(dirname string) ([]os.FileInfo, error)
ReadFile(filename string) (b []byte, err error)
Remove(name string) error
Rename(oldName, newName string) error
Stat(path string) (os.FileInfo, error)
}
// realOs is an implementation of Oser backed by the "os" package
type realOs struct {
}
// realOsFile is an implementation of vfs.Handle
type realOsFile struct {
*os.File
}
// Flush
func (f realOsFile) Flush() error {
return nil
}
// Release
func (f realOsFile) Release() error {
return f.File.Close()
}
// Node
func (f realOsFile) Node() vfs.Node {
return nil
}
// Chtimes
func (r realOs) Chtimes(name string, atime time.Time, mtime time.Time) error {
return os.Chtimes(name, atime, mtime)
}
// Create
func (r realOs) Create(name string) (vfs.Handle, error) {
fd, err := file.OpenFile(name, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0666)
if err != nil {
return nil, err
}
return realOsFile{File: fd}, err
}
// Mkdir
func (r realOs) Mkdir(name string, perm os.FileMode) error {
return os.Mkdir(name, perm)
}
// Open
func (r realOs) Open(name string) (vfs.Handle, error) {
fd, err := os.Open(name)
if err != nil {
return nil, err
}
return realOsFile{File: fd}, err
}
// OpenFile
func (r realOs) OpenFile(name string, flags int, perm os.FileMode) (vfs.Handle, error) {
fd, err := file.OpenFile(name, flags, perm)
if err != nil {
return nil, err
}
return realOsFile{File: fd}, err
}
// ReadDir
func (r realOs) ReadDir(dirname string) ([]os.FileInfo, error) {
return ioutil.ReadDir(dirname)
}
// ReadFile
func (r realOs) ReadFile(filename string) (b []byte, err error) {
return ioutil.ReadFile(filename)
}
// Remove
func (r realOs) Remove(name string) error {
return os.Remove(name)
}
// Rename
func (r realOs) Rename(oldName, newName string) error {
return os.Rename(oldName, newName)
}
// Stat
func (r realOs) Stat(path string) (os.FileInfo, error) {
return os.Stat(path)
}
// Check interfaces
var _ Oser = &realOs{}
var _ vfs.Handle = &realOsFile{}

11
cmd/mountlib/mounttest/read.go → vfs/vfstest/read.go
View file

@ -1,9 +1,8 @@
package mounttest
package vfstest
import (
"io"
"io/ioutil"
"os"
"testing"
"time"
@ -19,7 +18,7 @@ func TestReadByByte(t *testing.T) {
run.checkDir(t, "testfile 10")
for i := 0; i < len(data); i++ {
fd, err := os.Open(run.path("testfile"))
fd, err := run.os.Open(run.path("testfile"))
assert.NoError(t, err)
for j := 0; j < i; j++ {
buf := make([]byte, 1)
@ -50,7 +49,7 @@ func TestReadChecksum(t *testing.T) {
// The hash comparison would fail in Flush, if we did not
// ensure we read the whole file
fd, err := os.Open(run.path("bigfile"))
fd, err := run.os.Open(run.path("bigfile"))
assert.NoError(t, err)
buf := make([]byte, 10)
_, err = io.ReadFull(fd, buf)
@ -60,7 +59,7 @@ func TestReadChecksum(t *testing.T) {
// The hash comparison would fail, because we only read parts
// of the file
fd, err = os.Open(run.path("bigfile"))
fd, err = run.os.Open(run.path("bigfile"))
assert.NoError(t, err)
// read at start
_, err = io.ReadFull(fd, buf)
@ -85,7 +84,7 @@ func TestReadSeek(t *testing.T) {
run.createFile(t, "testfile", string(data))
run.checkDir(t, "testfile 10")
fd, err := os.Open(run.path("testfile"))
fd, err := run.os.Open(run.path("testfile"))
assert.NoError(t, err)
// Seek to half way

2
cmd/mountlib/mounttest/read_non_unix.go → vfs/vfstest/read_non_unix.go
View file

@ -1,6 +1,6 @@
// +build !linux,!darwin,!freebsd
package mounttest
package vfstest
import (
"runtime"

6
cmd/mountlib/mounttest/read_unix.go → vfs/vfstest/read_unix.go
View file

@ -1,9 +1,8 @@
// +build linux darwin freebsd
package mounttest
package vfstest
import (
"os"
"syscall"
"testing"
@ -12,11 +11,12 @@ import (
// TestReadFileDoubleClose tests double close on read
func TestReadFileDoubleClose(t *testing.T) {
run.skipIfVFS(t)
run.skipIfNoFUSE(t)
run.createFile(t, "testdoubleclose", "hello")
in, err := os.Open(run.path("testdoubleclose"))
in, err := run.os.Open(run.path("testdoubleclose"))
assert.NoError(t, err)
fd := in.Fd()

20
vfs/vfstest/vfs.go Normal file
View file

@ -0,0 +1,20 @@
package vfstest
import (
"os"
"github.com/rclone/rclone/vfs"
)
// vfsOs is an implementation of Oser backed by the "vfs" package
type vfsOs struct {
*vfs.VFS
}
// Stat
func (v vfsOs) Stat(path string) (os.FileInfo, error) {
return v.VFS.Stat(path)
}
// Check interfaces
var _ Oser = vfsOs{}

5
cmd/mountlib/mounttest/write.go → vfs/vfstest/write.go
View file

@ -1,4 +1,4 @@
package mounttest
package vfstest
import (
"os"
@ -89,6 +89,7 @@ func TestWriteFileFsync(t *testing.T) {
// TestWriteFileDup tests behavior of mmap() in Python by using dup() on a file handle
func TestWriteFileDup(t *testing.T) {
run.skipIfVFS(t)
run.skipIfNoFUSE(t)
if run.vfs.Opt.CacheMode < vfs.CacheModeWrites {
@ -169,7 +170,7 @@ func TestWriteFileAppend(t *testing.T) {
err = fh.Close()
require.NoError(t, err)
info, err := os.Stat(filepath)
info, err := run.os.Stat(filepath)
require.NoError(t, err)
require.EqualValues(t, len(testData)+len(appendData), info.Size())

2
cmd/mountlib/mounttest/write_non_unix.go → vfs/vfstest/write_non_unix.go
View file

@ -1,6 +1,6 @@
// +build !linux,!darwin,!freebsd
package mounttest
package vfstest
import (
"runtime"

3
cmd/mountlib/mounttest/write_unix.go → vfs/vfstest/write_unix.go
View file

@ -1,6 +1,6 @@
// +build linux darwin freebsd
package mounttest
package vfstest
import (
"runtime"
@ -14,6 +14,7 @@ import (
// TestWriteFileDoubleClose tests double close on write
func TestWriteFileDoubleClose(t *testing.T) {
run.skipIfVFS(t)
run.skipIfNoFUSE(t)
if runtime.GOOS == "darwin" {
t.Skip("Skipping test on OSX")

30
vfs/vfstest_test.go Normal file
View file

@ -0,0 +1,30 @@
// Run the more functional vfstest package on the vfs
package vfs_test
import (
"testing"
_ "github.com/rclone/rclone/backend/all" // import all the backends
"github.com/rclone/rclone/fs"
"github.com/rclone/rclone/fstest"
"github.com/rclone/rclone/vfs"
"github.com/rclone/rclone/vfs/vfstest"
)
// TestExt runs more functional tests all the tests against all the
// VFS cache modes
func TestFunctional(t *testing.T) {
if *fstest.RemoteName != "" {
t.Skip("Skip on non local")
}
vfstest.RunTests(t, true, func(f fs.Fs, mountpoint string) (VFS *vfs.VFS, unmountResult <-chan error, unmount func() error, err error) {
unmountResultChan := make(chan (error), 1)
unmount = func() error {
unmountResultChan <- nil
return nil
}
VFS = vfs.New(f, nil)
return VFS, unmountResultChan, unmount, nil
})
}