Make tests for each Fs

Factor tests out of rclonetest

drive/drive_test.go

@@ -0,0 +1,46 @@
+// Test Drive filesystem interface
+package drive_test
+
+import (
+	"testing"
+
+	"github.com/ncw/rclone/drive"
+	"github.com/ncw/rclone/fs"
+	"github.com/ncw/rclone/fstest/fstests"
+)
+
+func init() {
+	fstests.NilObject = fs.Object((*drive.FsObjectDrive)(nil))
+	fstests.RemoteName = "TestDrive:"
+}
+
+// Generic tests for the Fs
+func TestInit(t *testing.T)                  { fstests.TestInit(t) }
+func TestFsString(t *testing.T)              { fstests.TestFsString(t) }
+func TestFsRmdirEmpty(t *testing.T)          { fstests.TestFsRmdirEmpty(t) }
+func TestFsRmdirNotFound(t *testing.T)       { fstests.TestFsRmdirNotFound(t) }
+func TestFsMkdir(t *testing.T)               { fstests.TestFsMkdir(t) }
+func TestFsListEmpty(t *testing.T)           { fstests.TestFsListEmpty(t) }
+func TestFsListDirEmpty(t *testing.T)        { fstests.TestFsListDirEmpty(t) }
+func TestFsNewFsObjectNotFound(t *testing.T) { fstests.TestFsNewFsObjectNotFound(t) }
+func TestFsPutFile1(t *testing.T)            { fstests.TestFsPutFile1(t) }
+func TestFsPutFile2(t *testing.T)            { fstests.TestFsPutFile2(t) }
+func TestFsListDirFile2(t *testing.T)        { fstests.TestFsListDirFile2(t) }
+func TestFsListFile1(t *testing.T)           { fstests.TestFsListFile1(t) }
+func TestFsNewFsObject(t *testing.T)         { fstests.TestFsNewFsObject(t) }
+func TestFsListFile1and2(t *testing.T)       { fstests.TestFsListFile1and2(t) }
+func TestFsRmdirFull(t *testing.T)           { fstests.TestFsRmdirFull(t) }
+func TestFsPrecision(t *testing.T)           { fstests.TestFsPrecision(t) }
+func TestObjectString(t *testing.T)          { fstests.TestObjectString(t) }
+func TestObjectFs(t *testing.T)              { fstests.TestObjectFs(t) }
+func TestObjectRemote(t *testing.T)          { fstests.TestObjectRemote(t) }
+func TestObjectMd5sum(t *testing.T)          { fstests.TestObjectMd5sum(t) }
+func TestObjectModTime(t *testing.T)         { fstests.TestObjectModTime(t) }
+func TestObjectSetModTime(t *testing.T)      { fstests.TestObjectSetModTime(t) }
+func TestObjectSize(t *testing.T)            { fstests.TestObjectSize(t) }
+func TestObjectOpen(t *testing.T)            { fstests.TestObjectOpen(t) }
+func TestObjectUpdate(t *testing.T)          { fstests.TestObjectUpdate(t) }
+func TestObjectStorable(t *testing.T)        { fstests.TestObjectStorable(t) }
+func TestObjectRemove(t *testing.T)          { fstests.TestObjectRemove(t) }
+func TestObjectPurge(t *testing.T)           { fstests.TestObjectPurge(t) }
+func TestFinalise(t *testing.T)              { fstests.TestFinalise(t) }

dropbox/dropbox_test.go

@@ -0,0 +1,46 @@
+// Test Dropbox filesystem interface
+package dropbox_test
+
+import (
+	"testing"
+
+	"github.com/ncw/rclone/dropbox"
+	"github.com/ncw/rclone/fs"
+	"github.com/ncw/rclone/fstest/fstests"
+)
+
+func init() {
+	fstests.NilObject = fs.Object((*dropbox.FsObjectDropbox)(nil))
+	fstests.RemoteName = "TestDropbox:"
+}
+
+// Generic tests for the Fs
+func TestInit(t *testing.T)                  { fstests.TestInit(t) }
+func TestFsString(t *testing.T)              { fstests.TestFsString(t) }
+func TestFsRmdirEmpty(t *testing.T)          { fstests.TestFsRmdirEmpty(t) }
+func TestFsRmdirNotFound(t *testing.T)       { fstests.TestFsRmdirNotFound(t) }
+func TestFsMkdir(t *testing.T)               { fstests.TestFsMkdir(t) }
+func TestFsListEmpty(t *testing.T)           { fstests.TestFsListEmpty(t) }
+func TestFsListDirEmpty(t *testing.T)        { fstests.TestFsListDirEmpty(t) }
+func TestFsNewFsObjectNotFound(t *testing.T) { fstests.TestFsNewFsObjectNotFound(t) }
+func TestFsPutFile1(t *testing.T)            { fstests.TestFsPutFile1(t) }
+func TestFsPutFile2(t *testing.T)            { fstests.TestFsPutFile2(t) }
+func TestFsListDirFile2(t *testing.T)        { fstests.TestFsListDirFile2(t) }
+func TestFsListFile1(t *testing.T)           { fstests.TestFsListFile1(t) }
+func TestFsNewFsObject(t *testing.T)         { fstests.TestFsNewFsObject(t) }
+func TestFsListFile1and2(t *testing.T)       { fstests.TestFsListFile1and2(t) }
+func TestFsRmdirFull(t *testing.T)           { fstests.TestFsRmdirFull(t) }
+func TestFsPrecision(t *testing.T)           { fstests.TestFsPrecision(t) }
+func TestObjectString(t *testing.T)          { fstests.TestObjectString(t) }
+func TestObjectFs(t *testing.T)              { fstests.TestObjectFs(t) }
+func TestObjectRemote(t *testing.T)          { fstests.TestObjectRemote(t) }
+func TestObjectMd5sum(t *testing.T)          { fstests.TestObjectMd5sum(t) }
+func TestObjectModTime(t *testing.T)         { fstests.TestObjectModTime(t) }
+func TestObjectSetModTime(t *testing.T)      { fstests.TestObjectSetModTime(t) }
+func TestObjectSize(t *testing.T)            { fstests.TestObjectSize(t) }
+func TestObjectOpen(t *testing.T)            { fstests.TestObjectOpen(t) }
+func TestObjectUpdate(t *testing.T)          { fstests.TestObjectUpdate(t) }
+func TestObjectStorable(t *testing.T)        { fstests.TestObjectStorable(t) }
+func TestObjectRemove(t *testing.T)          { fstests.TestObjectRemove(t) }
+func TestObjectPurge(t *testing.T)           { fstests.TestObjectPurge(t) }
+func TestFinalise(t *testing.T)              { fstests.TestFinalise(t) }

fs/fs.go

@@ -79,9 +79,13 @@ type Fs interface {
 	Put(in io.Reader, remote string, modTime time.Time, size int64) (Object, error)
 
 	// Make the directory (container, bucket)
+	//
+	// Shouldn't return an error if it already exists
 	Mkdir() error
 
 	// Remove the directory (container, bucket) if empty
+	//
+	// Return an error if it doesn't exists or isn't empty
 	Rmdir() error
 
 	// Precision of the ModTimes in this Fs

fstest/fstest.go

@@ -0,0 +1,192 @@
+// Utilities for testing the fs
+
+package fstest
+
+// FIXME put name of test FS in Fs structure
+
+import (
+	"io/ioutil"
+	"log"
+	"math/rand"
+	"os"
+	"strings"
+	"time"
+
+	"github.com/ncw/rclone/fs"
+)
+
+var Fatalf = log.Fatalf
+
+// Represents an item for checking
+type Item struct {
+	Path    string
+	Md5sum  string
+	ModTime time.Time
+	Size    int64
+}
+
+func (i *Item) Check(obj fs.Object) {
+	if obj == nil {
+		Fatalf("Object is nil")
+	}
+	// Check attributes
+	Md5sum, err := obj.Md5sum()
+	if err != nil {
+		Fatalf("Failed to read md5sum for %q: %v", obj.Remote(), err)
+	}
+	if i.Md5sum != Md5sum {
+		Fatalf("%s: Md5sum incorrect - expecting %q got %q", obj.Remote(), i.Md5sum, Md5sum)
+	}
+	if i.Size != obj.Size() {
+		Fatalf("%s: Size incorrect - expecting %d got %d", obj.Remote(), i.Size, obj.Size())
+	}
+	// check the mod time to the given precision
+	modTime := obj.ModTime()
+	dt := modTime.Sub(i.ModTime)
+	if dt >= fs.Config.ModifyWindow || dt <= -fs.Config.ModifyWindow {
+		Fatalf("%s: Modification time difference too big |%s| > %s (%s vs %s)", obj.Remote(), dt, fs.Config.ModifyWindow, modTime, i.ModTime)
+	}
+
+}
+
+// Represents all items for checking
+type Items struct {
+	byName map[string]*Item
+	items  []Item
+}
+
+// Make an Items
+func NewItems(items []Item) *Items {
+	is := &Items{
+		byName: make(map[string]*Item),
+		items:  items,
+	}
+	// Fill up byName
+	for i := range items {
+		is.byName[items[i].Path] = &items[i]
+	}
+	return is
+}
+
+// Check off an item
+func (is *Items) Find(obj fs.Object) {
+	i, ok := is.byName[obj.Remote()]
+	if !ok {
+		Fatalf("Unexpected file %q", obj.Remote())
+	}
+	delete(is.byName, obj.Remote())
+	i.Check(obj)
+}
+
+// Check all done
+func (is *Items) Done() {
+	if len(is.byName) != 0 {
+		for name := range is.byName {
+			log.Printf("Not found %q", name)
+		}
+		Fatalf("%d objects not found", len(is.byName))
+	}
+}
+
+// Checks the fs to see if it has the expected contents
+func CheckListing(f fs.Fs, items []Item) {
+	is := NewItems(items)
+	for obj := range f.List() {
+		is.Find(obj)
+	}
+	is.Done()
+}
+
+// Parse a time string or explode
+func Time(timeString string) time.Time {
+	t, err := time.Parse(time.RFC3339Nano, timeString)
+	if err != nil {
+		Fatalf("Failed to parse time %q: %v", timeString, err)
+	}
+	return t
+}
+
+// Create a random string
+func RandomString(n int) string {
+	source := "abcdefghijklmnopqrstuvwxyz0123456789"
+	out := make([]byte, n)
+	for i := range out {
+		out[i] = source[rand.Intn(len(source))]
+	}
+	return string(out)
+}
+
+// Make a random bucket or subdirectory on the remote
+//
+// Call the finalise function returned to Purge the fs at the end (and
+// the parent if necessary)
+func RandomRemote(remoteName string, subdir bool) (fs.Fs, func()) {
+	// Make a directory if remote name is null
+	rmdir := ""
+	var err error
+	if remoteName == "" {
+		remoteName, err = ioutil.TempDir("", "rclone")
+		if err != nil {
+			Fatalf("Failed to create temp dir: %v", err)
+		}
+		rmdir = remoteName
+	}
+
+	if !strings.HasSuffix(remoteName, ":") {
+		remoteName += "/"
+	}
+	remoteName += RandomString(32)
+	var parentRemote fs.Fs
+	if subdir {
+		var err error
+		parentRemote, err = fs.NewFs(remoteName)
+		if err != nil {
+			log.Fatalf("Failed to make parent %q: %v", remoteName, err)
+		}
+		remoteName += "/" + RandomString(8)
+	}
+
+	remote, err := fs.NewFs(remoteName)
+	if err != nil {
+		log.Fatalf("Failed to make %q: %v", remoteName, err)
+	}
+
+	finalise := func() {
+		TestPurge(remote)
+		if parentRemote != nil {
+			TestPurge(parentRemote)
+		}
+		// Delete directory if we made one above
+		if rmdir != "" {
+			err := os.RemoveAll(rmdir)
+			if err != nil {
+				Fatalf("Failed to remove %q: %v", rmdir, err)
+			}
+		}
+	}
+
+	return remote, finalise
+}
+
+func TestMkdir(remote fs.Fs) {
+	err := fs.Mkdir(remote)
+	if err != nil {
+		Fatalf("Mkdir failed: %v", err)
+	}
+	CheckListing(remote, []Item{})
+}
+
+func TestPurge(remote fs.Fs) {
+	err := fs.Purge(remote)
+	if err != nil {
+		log.Fatalf("Purge failed: %v", err)
+	}
+	CheckListing(remote, []Item{})
+}
+
+func TestRmdir(remote fs.Fs) {
+	err := fs.Rmdir(remote)
+	if err != nil {
+		Fatalf("Rmdir failed: %v", err)
+	}
+}

fstest/fstests/fstests.go

@@ -0,0 +1,338 @@
+// Generic tests for testing the Fs and Object interfaces
+package fstests
+
+// FIXME need to check the limited file system
+
+import (
+	"bytes"
+	"crypto/md5"
+	"encoding/hex"
+	"io"
+	"testing"
+	"time"
+
+	"github.com/ncw/rclone/fs"
+	"github.com/ncw/rclone/fstest"
+)
+
+var (
+	remote         fs.Fs
+	RemoteName     = ""
+	remoteFinalise func()
+	NilObject      fs.Object
+	file1          = fstest.Item{
+		ModTime: fstest.Time("2001-02-03T04:05:06.499999999Z"),
+		Path:    "file name.txt",
+	}
+	file2 = fstest.Item{
+		ModTime: fstest.Time("2001-02-03T04:05:10.123123123Z"),
+		Path:    `hello? sausage/êé/Hello, 世界/ " ' @ < > & ?/z.txt`,
+	}
+)
+
+func TestInit(t *testing.T) {
+	fs.LoadConfig()
+	fs.Config.Verbose = false
+	fs.Config.Quiet = true
+	remote, remoteFinalise = fstest.RandomRemote(RemoteName, false)
+	// if err != nil {
+	// 	if strings.Contains(err.Error(), "Didn't find section in config file") {
+	// 		return
+	// 	}
+	// 	t.Fatalf("Couldn't start FS: %v", err)
+	// }
+	fstest.Fatalf = t.Fatalf
+	fstest.TestMkdir(remote)
+}
+
+func skipIfNotOk(t *testing.T) {
+	fstest.Fatalf = t.Fatalf
+	if remote == nil {
+		t.Skip("FS not configured")
+	}
+}
+
+// String returns a description of the FS
+
+func TestFsString(t *testing.T) {
+	skipIfNotOk(t)
+	str := remote.String()
+	if str == "" {
+		t.Fatal("Bad fs.String()")
+	}
+}
+
+type TestFile struct {
+	ModTime time.Time
+	Path    string
+	Size    int64
+	Md5sum  string
+}
+
+func TestFsRmdirEmpty(t *testing.T) {
+	skipIfNotOk(t)
+	fstest.TestRmdir(remote)
+}
+
+func TestFsRmdirNotFound(t *testing.T) {
+	skipIfNotOk(t)
+	err := remote.Rmdir()
+	if err == nil {
+		t.Fatalf("Expecting error on Rmdir non existent")
+	}
+}
+
+func TestFsMkdir(t *testing.T) {
+	skipIfNotOk(t)
+	fstest.TestMkdir(remote)
+	fstest.TestMkdir(remote)
+}
+
+func TestFsListEmpty(t *testing.T) {
+	skipIfNotOk(t)
+	fstest.CheckListing(remote, []fstest.Item{})
+}
+
+func TestFsListDirEmpty(t *testing.T) {
+	skipIfNotOk(t)
+	for obj := range remote.ListDir() {
+		t.Error("Found unexpected item %q", obj.Name)
+	}
+}
+
+func TestFsNewFsObjectNotFound(t *testing.T) {
+	skipIfNotOk(t)
+	if remote.NewFsObject("potato") != nil {
+		t.Fatal("Didn't expect to find object")
+	}
+}
+
+func findObject(t *testing.T, Name string) fs.Object {
+	obj := remote.NewFsObject(Name)
+	if obj == nil {
+		t.Fatalf("nil object")
+	}
+	return obj
+}
+
+func testPut(t *testing.T, file *fstest.Item) {
+	buf := bytes.NewBufferString(fstest.RandomString(100))
+	hash := md5.New()
+	in := io.TeeReader(buf, hash)
+
+	file.Size = int64(buf.Len())
+	obj, err := remote.Put(in, file.Path, file.ModTime, file.Size)
+	if err != nil {
+		t.Fatal("Put error", err)
+	}
+	file.Md5sum = hex.EncodeToString(hash.Sum(nil))
+	file.Check(obj)
+	// Re-read the object and check again
+	obj = findObject(t, file.Path)
+	file.Check(obj)
+}
+
+func TestFsPutFile1(t *testing.T) {
+	skipIfNotOk(t)
+	testPut(t, &file1)
+}
+
+func TestFsPutFile2(t *testing.T) {
+	skipIfNotOk(t)
+	testPut(t, &file2)
+}
+
+func TestFsListDirFile2(t *testing.T) {
+	skipIfNotOk(t)
+	found := false
+	for obj := range remote.ListDir() {
+		if obj.Name != `hello? sausage` {
+			t.Errorf("Found unexpected item %q", obj.Name)
+		} else {
+			found = true
+		}
+	}
+	if !found {
+		t.Errorf("Didn't find %q", `hello? sausage`)
+	}
+}
+
+func TestFsListFile1(t *testing.T) {
+	skipIfNotOk(t)
+	fstest.CheckListing(remote, []fstest.Item{file1, file2})
+}
+
+func TestFsNewFsObject(t *testing.T) {
+	skipIfNotOk(t)
+	obj := findObject(t, file1.Path)
+	file1.Check(obj)
+}
+
+func TestFsListFile1and2(t *testing.T) {
+	skipIfNotOk(t)
+	fstest.CheckListing(remote, []fstest.Item{file1, file2})
+}
+
+func TestFsRmdirFull(t *testing.T) {
+	skipIfNotOk(t)
+	err := remote.Rmdir()
+	if err == nil {
+		t.Fatalf("Expecting error on RMdir on non empty remote")
+	}
+}
+
+func TestFsPrecision(t *testing.T) {
+	skipIfNotOk(t)
+	precision := remote.Precision()
+	if precision > time.Second || precision < 0 {
+		t.Fatalf("Precision out of range %v", precision)
+	}
+	// FIXME check expected precision
+}
+
+func TestObjectString(t *testing.T) {
+	skipIfNotOk(t)
+	obj := findObject(t, file1.Path)
+	s := obj.String()
+	if s != file1.Path {
+		t.Errorf("String() wrong %v != %v", s, file1.Path)
+	}
+	obj = NilObject
+	s = obj.String()
+	if s != "<nil>" {
+		t.Errorf("String() wrong %v != %v", s, "<nil>")
+	}
+}
+
+func TestObjectFs(t *testing.T) {
+	skipIfNotOk(t)
+	obj := findObject(t, file1.Path)
+	if obj.Fs() != remote {
+		t.Errorf("Fs is wrong %v != %v", obj.Fs(), remote)
+	}
+}
+
+func TestObjectRemote(t *testing.T) {
+	skipIfNotOk(t)
+	obj := findObject(t, file1.Path)
+	if obj.Remote() != file1.Path {
+		t.Errorf("Remote is wrong %v != %v", obj.Remote(), file1.Path)
+	}
+}
+
+func TestObjectMd5sum(t *testing.T) {
+	skipIfNotOk(t)
+	obj := findObject(t, file1.Path)
+	Md5sum, err := obj.Md5sum()
+	if err != nil {
+		t.Errorf("Error in Md5sum: %v", err)
+	}
+	if Md5sum != file1.Md5sum {
+		t.Errorf("Md5sum is wrong %v != %v", Md5sum, file1.Md5sum)
+	}
+}
+
+func TestObjectModTime(t *testing.T) {
+	skipIfNotOk(t)
+	obj := findObject(t, file1.Path)
+	if !obj.ModTime().Equal(file1.ModTime) {
+		t.Errorf("ModTime is wrong %v != %v", obj.ModTime(), file1.ModTime)
+	}
+}
+
+func TestObjectSetModTime(t *testing.T) {
+	skipIfNotOk(t)
+	newModTime := fstest.Time("2011-12-13T14:15:16.999999999Z")
+	obj := findObject(t, file1.Path)
+	obj.SetModTime(newModTime)
+	// Check in this object
+	if !obj.ModTime().Equal(newModTime) {
+		t.Errorf("newModTime is wrong %v != %v", obj.ModTime(), newModTime)
+	}
+	file1.ModTime = newModTime
+	// And make a new object and read it from there too
+	TestObjectModTime(t)
+}
+
+func TestObjectSize(t *testing.T) {
+	skipIfNotOk(t)
+	obj := findObject(t, file1.Path)
+	if obj.Size() != file1.Size {
+		t.Errorf("Size is wrong %v != %v", obj.Size(), file1.Size)
+	}
+}
+
+func TestObjectOpen(t *testing.T) {
+	skipIfNotOk(t)
+	obj := findObject(t, file1.Path)
+	in, err := obj.Open()
+	if err != nil {
+		t.Fatalf("Open() return error: %v", err)
+	}
+	hash := md5.New()
+	n, err := io.Copy(hash, in)
+	if err != nil {
+		t.Fatalf("io.Copy() return error: %v", err)
+	}
+	if n != file1.Size {
+		t.Fatalf("Read wrong number of bytes %d != %d", n, file1.Size)
+	}
+	err = in.Close()
+	if err != nil {
+		t.Fatalf("in.Close() return error: %v", err)
+	}
+	Md5sum := hex.EncodeToString(hash.Sum(nil))
+	if Md5sum != file1.Md5sum {
+		t.Errorf("Md5sum is wrong %v != %v", Md5sum, file1.Md5sum)
+	}
+}
+
+func TestObjectUpdate(t *testing.T) {
+	skipIfNotOk(t)
+	buf := bytes.NewBufferString(fstest.RandomString(200))
+	hash := md5.New()
+	in := io.TeeReader(buf, hash)
+
+	file1.Size = int64(buf.Len())
+	obj := findObject(t, file1.Path)
+	err := obj.Update(in, file1.ModTime, file1.Size)
+	if err != nil {
+		t.Fatal("Update error", err)
+	}
+	file1.Md5sum = hex.EncodeToString(hash.Sum(nil))
+	file1.Check(obj)
+	// Re-read the object and check again
+	obj = findObject(t, file1.Path)
+	file1.Check(obj)
+}
+
+func TestObjectStorable(t *testing.T) {
+	skipIfNotOk(t)
+	obj := findObject(t, file1.Path)
+	if !obj.Storable() {
+		t.Fatalf("Expecting %v to be storable", obj)
+	}
+}
+
+func TestObjectRemove(t *testing.T) {
+	skipIfNotOk(t)
+	obj := findObject(t, file1.Path)
+	err := obj.Remove()
+	if err != nil {
+		t.Fatal("Remove error", err)
+	}
+	fstest.CheckListing(remote, []fstest.Item{file2})
+}
+
+func TestObjectPurge(t *testing.T) {
+	skipIfNotOk(t)
+	fstest.TestPurge(remote)
+	fstest.TestPurge(remote)
+}
+
+func TestFinalise(t *testing.T) {
+	skipIfNotOk(t)
+	if remoteFinalise != nil {
+		remoteFinalise()
+	}
+}

googlecloudstorage/googlecloudstorage_test.go

@@ -0,0 +1,46 @@
+// Test Google cloud storage filesystem interface
+package googlecloudstorage_test
+
+import (
+	"testing"
+
+	"github.com/ncw/rclone/fs"
+	"github.com/ncw/rclone/fstest/fstests"
+	"github.com/ncw/rclone/googlecloudstorage"
+)
+
+func init() {
+	fstests.NilObject = fs.Object((*googlecloudstorage.FsObjectStorage)(nil))
+	fstests.RemoteName = "TestGoogleCloudStorage:"
+}
+
+// Generic tests for the Fs
+func TestInit(t *testing.T)                  { fstests.TestInit(t) }
+func TestFsString(t *testing.T)              { fstests.TestFsString(t) }
+func TestFsRmdirEmpty(t *testing.T)          { fstests.TestFsRmdirEmpty(t) }
+func TestFsRmdirNotFound(t *testing.T)       { fstests.TestFsRmdirNotFound(t) }
+func TestFsMkdir(t *testing.T)               { fstests.TestFsMkdir(t) }
+func TestFsListEmpty(t *testing.T)           { fstests.TestFsListEmpty(t) }
+func TestFsListDirEmpty(t *testing.T)        { fstests.TestFsListDirEmpty(t) }
+func TestFsNewFsObjectNotFound(t *testing.T) { fstests.TestFsNewFsObjectNotFound(t) }
+func TestFsPutFile1(t *testing.T)            { fstests.TestFsPutFile1(t) }
+func TestFsPutFile2(t *testing.T)            { fstests.TestFsPutFile2(t) }
+func TestFsListDirFile2(t *testing.T)        { fstests.TestFsListDirFile2(t) }
+func TestFsListFile1(t *testing.T)           { fstests.TestFsListFile1(t) }
+func TestFsNewFsObject(t *testing.T)         { fstests.TestFsNewFsObject(t) }
+func TestFsListFile1and2(t *testing.T)       { fstests.TestFsListFile1and2(t) }
+func TestFsRmdirFull(t *testing.T)           { fstests.TestFsRmdirFull(t) }
+func TestFsPrecision(t *testing.T)           { fstests.TestFsPrecision(t) }
+func TestObjectString(t *testing.T)          { fstests.TestObjectString(t) }
+func TestObjectFs(t *testing.T)              { fstests.TestObjectFs(t) }
+func TestObjectRemote(t *testing.T)          { fstests.TestObjectRemote(t) }
+func TestObjectMd5sum(t *testing.T)          { fstests.TestObjectMd5sum(t) }
+func TestObjectModTime(t *testing.T)         { fstests.TestObjectModTime(t) }
+func TestObjectSetModTime(t *testing.T)      { fstests.TestObjectSetModTime(t) }
+func TestObjectSize(t *testing.T)            { fstests.TestObjectSize(t) }
+func TestObjectOpen(t *testing.T)            { fstests.TestObjectOpen(t) }
+func TestObjectUpdate(t *testing.T)          { fstests.TestObjectUpdate(t) }
+func TestObjectStorable(t *testing.T)        { fstests.TestObjectStorable(t) }
+func TestObjectRemove(t *testing.T)          { fstests.TestObjectRemove(t) }
+func TestObjectPurge(t *testing.T)           { fstests.TestObjectPurge(t) }
+func TestFinalise(t *testing.T)              { fstests.TestFinalise(t) }

local/local_test.go

@@ -0,0 +1,46 @@
+// Test Local filesystem interface
+package local_test
+
+import (
+	"testing"
+
+	"github.com/ncw/rclone/fs"
+	"github.com/ncw/rclone/fstest/fstests"
+	"github.com/ncw/rclone/local"
+)
+
+func init() {
+	fstests.NilObject = fs.Object((*local.FsObjectLocal)(nil))
+	fstests.RemoteName = ""
+}
+
+// Generic tests for the Fs
+func TestInit(t *testing.T)                  { fstests.TestInit(t) }
+func TestFsString(t *testing.T)              { fstests.TestFsString(t) }
+func TestFsRmdirEmpty(t *testing.T)          { fstests.TestFsRmdirEmpty(t) }
+func TestFsRmdirNotFound(t *testing.T)       { fstests.TestFsRmdirNotFound(t) }
+func TestFsMkdir(t *testing.T)               { fstests.TestFsMkdir(t) }
+func TestFsListEmpty(t *testing.T)           { fstests.TestFsListEmpty(t) }
+func TestFsListDirEmpty(t *testing.T)        { fstests.TestFsListDirEmpty(t) }
+func TestFsNewFsObjectNotFound(t *testing.T) { fstests.TestFsNewFsObjectNotFound(t) }
+func TestFsPutFile1(t *testing.T)            { fstests.TestFsPutFile1(t) }
+func TestFsPutFile2(t *testing.T)            { fstests.TestFsPutFile2(t) }
+func TestFsListDirFile2(t *testing.T)        { fstests.TestFsListDirFile2(t) }
+func TestFsListFile1(t *testing.T)           { fstests.TestFsListFile1(t) }
+func TestFsNewFsObject(t *testing.T)         { fstests.TestFsNewFsObject(t) }
+func TestFsListFile1and2(t *testing.T)       { fstests.TestFsListFile1and2(t) }
+func TestFsRmdirFull(t *testing.T)           { fstests.TestFsRmdirFull(t) }
+func TestFsPrecision(t *testing.T)           { fstests.TestFsPrecision(t) }
+func TestObjectString(t *testing.T)          { fstests.TestObjectString(t) }
+func TestObjectFs(t *testing.T)              { fstests.TestObjectFs(t) }
+func TestObjectRemote(t *testing.T)          { fstests.TestObjectRemote(t) }
+func TestObjectMd5sum(t *testing.T)          { fstests.TestObjectMd5sum(t) }
+func TestObjectModTime(t *testing.T)         { fstests.TestObjectModTime(t) }
+func TestObjectSetModTime(t *testing.T)      { fstests.TestObjectSetModTime(t) }
+func TestObjectSize(t *testing.T)            { fstests.TestObjectSize(t) }
+func TestObjectOpen(t *testing.T)            { fstests.TestObjectOpen(t) }
+func TestObjectUpdate(t *testing.T)          { fstests.TestObjectUpdate(t) }
+func TestObjectStorable(t *testing.T)        { fstests.TestObjectStorable(t) }
+func TestObjectRemove(t *testing.T)          { fstests.TestObjectRemove(t) }
+func TestObjectPurge(t *testing.T)           { fstests.TestObjectPurge(t) }
+func TestFinalise(t *testing.T)              { fstests.TestFinalise(t) }

rclonetest/rclonetest.go

@@ -10,10 +10,10 @@ import (
 	"math/rand"
 	"os"
 	"path"
-	"strings"
 	"time"
 
 	"github.com/ncw/rclone/fs"
+	"github.com/ncw/rclone/fstest"
 	"github.com/ogier/pflag"
 
 	// Active file systems
@@ -32,88 +32,6 @@ var (
 	subDir                = pflag.BoolP("subdir", "S", false, "Test with a sub directory")
 )
 
-// Represents an item for checking
-type Item struct {
-	Path    string
-	Md5sum  string
-	ModTime time.Time
-	Size    int64
-}
-
-// Represents all items for checking
-type Items struct {
-	byName map[string]*Item
-	items  []Item
-}
-
-// Make an Items
-func NewItems(items []Item) *Items {
-	is := &Items{
-		byName: make(map[string]*Item),
-		items:  items,
-	}
-	// Fill up byName
-	for i := range items {
-		is.byName[items[i].Path] = &items[i]
-	}
-	return is
-}
-
-// Check off an item
-func (is *Items) Find(obj fs.Object) {
-	i, ok := is.byName[obj.Remote()]
-	if !ok {
-		log.Fatalf("Unexpected file %q", obj.Remote())
-	}
-	delete(is.byName, obj.Remote())
-	// Check attributes
-	Md5sum, err := obj.Md5sum()
-	if err != nil {
-		log.Fatalf("Failed to read md5sum for %q: %v", obj.Remote(), err)
-	}
-	if i.Md5sum != Md5sum {
-		log.Fatalf("%s: Md5sum incorrect - expecting %q got %q", obj.Remote(), i.Md5sum, Md5sum)
-	}
-	if i.Size != obj.Size() {
-		log.Fatalf("%s: Size incorrect - expecting %d got %d", obj.Remote(), i.Size, obj.Size())
-	}
-	// check the mod time to the given precision
-	modTime := obj.ModTime()
-	dt := modTime.Sub(i.ModTime)
-	if dt >= fs.Config.ModifyWindow || dt <= -fs.Config.ModifyWindow {
-		log.Fatalf("%s: Modification time difference too big |%s| > %s (%s vs %s)", obj.Remote(), dt, fs.Config.ModifyWindow, modTime, i.ModTime)
-	}
-
-}
-
-// Check all done
-func (is *Items) Done() {
-	if len(is.byName) != 0 {
-		for name := range is.byName {
-			log.Printf("Not found %q", name)
-		}
-		log.Fatalf("%d objects not found", len(is.byName))
-	}
-}
-
-// Checks the fs to see if it has the expected contents
-func CheckListing(f fs.Fs, items []Item) {
-	is := NewItems(items)
-	for obj := range f.List() {
-		is.Find(obj)
-	}
-	is.Done()
-}
-
-// Parse a time string or explode
-func Time(timeString string) time.Time {
-	t, err := time.Parse(time.RFC3339Nano, timeString)
-	if err != nil {
-		log.Fatalf("Failed to parse time %q: %v", timeString, err)
-	}
-	return t
-}
-
 // Write a file
 func WriteFile(filePath, content string, t time.Time) {
 	// FIXME make directories?
@@ -133,29 +51,9 @@ func WriteFile(filePath, content string, t time.Time) {
 	}
 }
 
-// Create a random string
+var t1 = fstest.Time("2001-02-03T04:05:06.499999999Z")
-func RandomString(n int) string {
+var t2 = fstest.Time("2011-12-25T12:59:59.123456789Z")
-	source := "abcdefghijklmnopqrstuvwxyz0123456789"
+var t3 = fstest.Time("2011-12-30T12:59:59.000000000Z")
-	out := make([]byte, n)
-	for i := range out {
-		out[i] = source[rand.Intn(len(source))]
-	}
-	return string(out)
-}
-
-func TestMkdir(flocal, fremote fs.Fs) {
-	err := fs.Mkdir(fremote)
-	if err != nil {
-		log.Fatalf("Mkdir failed: %v", err)
-	}
-	items := []Item{}
-	CheckListing(flocal, items)
-	CheckListing(fremote, items)
-}
-
-var t1 = Time("2001-02-03T04:05:06.499999999Z")
-var t2 = Time("2011-12-25T12:59:59.123456789Z")
-var t3 = Time("2011-12-30T12:59:59.000000000Z")
 
 func TestCopy(flocal, fremote fs.Fs) {
 	WriteFile("sub dir/hello world", "hello world", t1)
@@ -169,12 +67,12 @@ func TestCopy(flocal, fremote fs.Fs) {
 		log.Fatalf("Copy failed: %v", err)
 	}
 
-	items := []Item{
+	items := []fstest.Item{
 		{Path: "sub dir/hello world", Size: 11, ModTime: t1, Md5sum: "5eb63bbbe01eeed093cb22bb8f5acdc3"},
 	}
 
-	CheckListing(flocal, items)
+	fstest.CheckListing(flocal, items)
-	CheckListing(fremote, []Item{})
+	fstest.CheckListing(fremote, []fstest.Item{})
 
 	// Now without dry run
 
@@ -184,8 +82,8 @@ func TestCopy(flocal, fremote fs.Fs) {
 		log.Fatalf("Copy failed: %v", err)
 	}
 
-	CheckListing(flocal, items)
+	fstest.CheckListing(flocal, items)
-	CheckListing(fremote, items)
+	fstest.CheckListing(fremote, items)
 
 	// Now delete the local file and download it
 
@@ -194,8 +92,8 @@ func TestCopy(flocal, fremote fs.Fs) {
 		log.Fatalf("Remove failed: %v", err)
 	}
 
-	CheckListing(flocal, []Item{})
+	fstest.CheckListing(flocal, []fstest.Item{})
-	CheckListing(fremote, items)
+	fstest.CheckListing(fremote, items)
 
 	log.Printf("Copy - redownload")
 	err = fs.Sync(flocal, fremote, false)
@@ -203,8 +101,8 @@ func TestCopy(flocal, fremote fs.Fs) {
 		log.Fatalf("Copy failed: %v", err)
 	}
 
-	CheckListing(flocal, items)
+	fstest.CheckListing(flocal, items)
-	CheckListing(fremote, items)
+	fstest.CheckListing(fremote, items)
 
 	// Clean the directory
 	cleanTempDir()
@@ -222,11 +120,11 @@ func TestSync(flocal, fremote fs.Fs) {
 	if err != nil {
 		log.Fatalf("Sync failed: %v", err)
 	}
-	items := []Item{
+	items := []fstest.Item{
 		{Path: "empty space", Size: 0, ModTime: t2, Md5sum: "d41d8cd98f00b204e9800998ecf8427e"},
 	}
-	CheckListing(flocal, items)
+	fstest.CheckListing(flocal, items)
-	CheckListing(fremote, items)
+	fstest.CheckListing(fremote, items)
 
 	// ------------------------------------------------------------
 
@@ -236,12 +134,12 @@ func TestSync(flocal, fremote fs.Fs) {
 	if err != nil {
 		log.Fatalf("Sync failed: %v", err)
 	}
-	items = []Item{
+	items = []fstest.Item{
 		{Path: "empty space", Size: 0, ModTime: t2, Md5sum: "d41d8cd98f00b204e9800998ecf8427e"},
 		{Path: "potato", Size: 60, ModTime: t3, Md5sum: "d6548b156ea68a4e003e786df99eee76"},
 	}
-	CheckListing(flocal, items)
+	fstest.CheckListing(flocal, items)
-	CheckListing(fremote, items)
+	fstest.CheckListing(fremote, items)
 
 	// ------------------------------------------------------------
 
@@ -251,12 +149,12 @@ func TestSync(flocal, fremote fs.Fs) {
 	if err != nil {
 		log.Fatalf("Sync failed: %v", err)
 	}
-	items = []Item{
+	items = []fstest.Item{
 		{Path: "empty space", Size: 0, ModTime: t2, Md5sum: "d41d8cd98f00b204e9800998ecf8427e"},
 		{Path: "potato", Size: 21, ModTime: t3, Md5sum: "100defcf18c42a1e0dc42a789b107cd2"},
 	}
-	CheckListing(flocal, items)
+	fstest.CheckListing(flocal, items)
-	CheckListing(fremote, items)
+	fstest.CheckListing(fremote, items)
 
 	// ------------------------------------------------------------
 
@@ -266,12 +164,12 @@ func TestSync(flocal, fremote fs.Fs) {
 	if err != nil {
 		log.Fatalf("Sync failed: %v", err)
 	}
-	items = []Item{
+	items = []fstest.Item{
 		{Path: "empty space", Size: 0, ModTime: t2, Md5sum: "d41d8cd98f00b204e9800998ecf8427e"},
 		{Path: "potato", Size: 21, ModTime: t2, Md5sum: "e4cb6955d9106df6263c45fcfc10f163"},
 	}
-	CheckListing(flocal, items)
+	fstest.CheckListing(flocal, items)
-	CheckListing(fremote, items)
+	fstest.CheckListing(fremote, items)
 
 	// ------------------------------------------------------------
 
@@ -288,24 +186,24 @@ func TestSync(flocal, fremote fs.Fs) {
 		log.Fatalf("Sync failed: %v", err)
 	}
 
-	before := []Item{
+	before := []fstest.Item{
 		{Path: "empty space", Size: 0, ModTime: t2, Md5sum: "d41d8cd98f00b204e9800998ecf8427e"},
 		{Path: "potato", Size: 21, ModTime: t2, Md5sum: "e4cb6955d9106df6263c45fcfc10f163"},
 	}
-	items = []Item{
+	items = []fstest.Item{
 		{Path: "empty space", Size: 0, ModTime: t2, Md5sum: "d41d8cd98f00b204e9800998ecf8427e"},
 		{Path: "potato2", Size: 60, ModTime: t1, Md5sum: "d6548b156ea68a4e003e786df99eee76"},
 	}
-	CheckListing(flocal, items)
+	fstest.CheckListing(flocal, items)
-	CheckListing(fremote, before)
+	fstest.CheckListing(fremote, before)
 
 	log.Printf("Sync after removing a file and adding a file")
 	err = fs.Sync(fremote, flocal, true)
 	if err != nil {
 		log.Fatalf("Sync failed: %v", err)
 	}
-	CheckListing(flocal, items)
+	fstest.CheckListing(flocal, items)
-	CheckListing(fremote, items)
+	fstest.CheckListing(fremote, items)
 }
 
 func TestLs(flocal, fremote fs.Fs) {
@@ -322,28 +220,6 @@ func TestLsd(flocal, fremote fs.Fs) {
 func TestCheck(flocal, fremote fs.Fs) {
 }
 
-func TestPurge(fremote fs.Fs) {
-	err := fs.Purge(fremote)
-	if err != nil {
-		log.Fatalf("Purge failed: %v", err)
-	}
-	unexpected := 0
-	for obj := range fremote.List() {
-		unexpected++
-		log.Printf("Found unexpected item %s", obj.Remote())
-	}
-	if unexpected != 0 {
-		log.Fatalf("exiting as found %d unexpected items", unexpected)
-	}
-}
-
-func TestRmdir(flocal, fremote fs.Fs) {
-	err := fs.Rmdir(fremote)
-	if err != nil {
-		log.Fatalf("Rmdir failed: %v", err)
-	}
-}
-
 func syntaxError() {
 	fmt.Fprintf(os.Stderr, `Test rclone with a remote to find bugs in either - %s.
 
@@ -383,32 +259,15 @@ func main() {
 		os.Exit(1)
 	}
 
-	remoteName = args[0]
+	fremote, finalise := fstest.RandomRemote(args[0], *subDir)
-	if !strings.HasSuffix(remoteName, ":") {
+	log.Printf("Testing with remote %v", fremote)
-		remoteName += "/"
+
-	}
-	remoteName += RandomString(32)
-	var parentRemote fs.Fs
-	if *subDir {
-		var err error
-		parentRemote, err = fs.NewFs(remoteName)
-		if err != nil {
-			log.Fatalf("Failed to make parent %q: %v", remoteName, err)
-		}
-		remoteName += "/" + RandomString(8)
-	}
-	log.Printf("Testing with remote %q", remoteName)
 	var err error
 	localName, err = ioutil.TempDir("", "rclone")
 	if err != nil {
 		log.Fatalf("Failed to create temp dir: %v", err)
 	}
 	log.Printf("Testing with local %q", localName)
-
-	fremote, err := fs.NewFs(remoteName)
-	if err != nil {
-		log.Fatalf("Failed to make %q: %v", remoteName, err)
-	}
 	flocal, err := fs.NewFs(localName)
 	if err != nil {
 		log.Fatalf("Failed to make %q: %v", remoteName, err)
@@ -416,18 +275,15 @@ func main() {
 
 	fs.CalculateModifyWindow(fremote, flocal)
 
-	TestMkdir(flocal, fremote)
+	fstest.TestMkdir(fremote)
 	TestCopy(flocal, fremote)
 	TestSync(flocal, fremote)
 	TestLs(flocal, fremote)
 	TestLsd(flocal, fremote)
 	TestCheck(flocal, fremote)
-	TestPurge(fremote)
 	//TestRmdir(flocal, fremote)
 
-	if parentRemote != nil {
+	finalise()
-		TestPurge(parentRemote)
-	}
 
 	cleanTempDir()
 	log.Printf("Tests OK")

rclonetest/test.sh

@@ -3,11 +3,11 @@
 go install
 
 REMOTES="
-memstore:
+TestSwift:
-s3:
+TestS3:
-drive2:
+TestDrive:
-gcs:
+TestGoogleCloudStorage:
-dropbox:
+TestDropbox:
 /tmp/z
 "
 

s3/s3_test.go

@@ -0,0 +1,46 @@
+// Test S3 filesystem interface
+package s3_test
+
+import (
+	"testing"
+
+	"github.com/ncw/rclone/fs"
+	"github.com/ncw/rclone/fstest/fstests"
+	"github.com/ncw/rclone/s3"
+)
+
+func init() {
+	fstests.NilObject = fs.Object((*s3.FsObjectS3)(nil))
+	fstests.RemoteName = "TestS3:"
+}
+
+// Generic tests for the Fs
+func TestInit(t *testing.T)                  { fstests.TestInit(t) }
+func TestFsString(t *testing.T)              { fstests.TestFsString(t) }
+func TestFsRmdirEmpty(t *testing.T)          { fstests.TestFsRmdirEmpty(t) }
+func TestFsRmdirNotFound(t *testing.T)       { fstests.TestFsRmdirNotFound(t) }
+func TestFsMkdir(t *testing.T)               { fstests.TestFsMkdir(t) }
+func TestFsListEmpty(t *testing.T)           { fstests.TestFsListEmpty(t) }
+func TestFsListDirEmpty(t *testing.T)        { fstests.TestFsListDirEmpty(t) }
+func TestFsNewFsObjectNotFound(t *testing.T) { fstests.TestFsNewFsObjectNotFound(t) }
+func TestFsPutFile1(t *testing.T)            { fstests.TestFsPutFile1(t) }
+func TestFsPutFile2(t *testing.T)            { fstests.TestFsPutFile2(t) }
+func TestFsListDirFile2(t *testing.T)        { fstests.TestFsListDirFile2(t) }
+func TestFsListFile1(t *testing.T)           { fstests.TestFsListFile1(t) }
+func TestFsNewFsObject(t *testing.T)         { fstests.TestFsNewFsObject(t) }
+func TestFsListFile1and2(t *testing.T)       { fstests.TestFsListFile1and2(t) }
+func TestFsRmdirFull(t *testing.T)           { fstests.TestFsRmdirFull(t) }
+func TestFsPrecision(t *testing.T)           { fstests.TestFsPrecision(t) }
+func TestObjectString(t *testing.T)          { fstests.TestObjectString(t) }
+func TestObjectFs(t *testing.T)              { fstests.TestObjectFs(t) }
+func TestObjectRemote(t *testing.T)          { fstests.TestObjectRemote(t) }
+func TestObjectMd5sum(t *testing.T)          { fstests.TestObjectMd5sum(t) }
+func TestObjectModTime(t *testing.T)         { fstests.TestObjectModTime(t) }
+func TestObjectSetModTime(t *testing.T)      { fstests.TestObjectSetModTime(t) }
+func TestObjectSize(t *testing.T)            { fstests.TestObjectSize(t) }
+func TestObjectOpen(t *testing.T)            { fstests.TestObjectOpen(t) }
+func TestObjectUpdate(t *testing.T)          { fstests.TestObjectUpdate(t) }
+func TestObjectStorable(t *testing.T)        { fstests.TestObjectStorable(t) }
+func TestObjectRemove(t *testing.T)          { fstests.TestObjectRemove(t) }
+func TestObjectPurge(t *testing.T)           { fstests.TestObjectPurge(t) }
+func TestFinalise(t *testing.T)              { fstests.TestFinalise(t) }

swift/swift_test.go

@@ -0,0 +1,46 @@
+// Test Swift filesystem interface
+package swift_test
+
+import (
+	"testing"
+
+	"github.com/ncw/rclone/fs"
+	"github.com/ncw/rclone/fstest/fstests"
+	"github.com/ncw/rclone/swift"
+)
+
+func init() {
+	fstests.NilObject = fs.Object((*swift.FsObjectSwift)(nil))
+	fstests.RemoteName = "TestSwift:"
+}
+
+// Generic tests for the Fs
+func TestInit(t *testing.T)                  { fstests.TestInit(t) }
+func TestFsString(t *testing.T)              { fstests.TestFsString(t) }
+func TestFsRmdirEmpty(t *testing.T)          { fstests.TestFsRmdirEmpty(t) }
+func TestFsRmdirNotFound(t *testing.T)       { fstests.TestFsRmdirNotFound(t) }
+func TestFsMkdir(t *testing.T)               { fstests.TestFsMkdir(t) }
+func TestFsListEmpty(t *testing.T)           { fstests.TestFsListEmpty(t) }
+func TestFsListDirEmpty(t *testing.T)        { fstests.TestFsListDirEmpty(t) }
+func TestFsNewFsObjectNotFound(t *testing.T) { fstests.TestFsNewFsObjectNotFound(t) }
+func TestFsPutFile1(t *testing.T)            { fstests.TestFsPutFile1(t) }
+func TestFsPutFile2(t *testing.T)            { fstests.TestFsPutFile2(t) }
+func TestFsListDirFile2(t *testing.T)        { fstests.TestFsListDirFile2(t) }
+func TestFsListFile1(t *testing.T)           { fstests.TestFsListFile1(t) }
+func TestFsNewFsObject(t *testing.T)         { fstests.TestFsNewFsObject(t) }
+func TestFsListFile1and2(t *testing.T)       { fstests.TestFsListFile1and2(t) }
+func TestFsRmdirFull(t *testing.T)           { fstests.TestFsRmdirFull(t) }
+func TestFsPrecision(t *testing.T)           { fstests.TestFsPrecision(t) }
+func TestObjectString(t *testing.T)          { fstests.TestObjectString(t) }
+func TestObjectFs(t *testing.T)              { fstests.TestObjectFs(t) }
+func TestObjectRemote(t *testing.T)          { fstests.TestObjectRemote(t) }
+func TestObjectMd5sum(t *testing.T)          { fstests.TestObjectMd5sum(t) }
+func TestObjectModTime(t *testing.T)         { fstests.TestObjectModTime(t) }
+func TestObjectSetModTime(t *testing.T)      { fstests.TestObjectSetModTime(t) }
+func TestObjectSize(t *testing.T)            { fstests.TestObjectSize(t) }
+func TestObjectOpen(t *testing.T)            { fstests.TestObjectOpen(t) }
+func TestObjectUpdate(t *testing.T)          { fstests.TestObjectUpdate(t) }
+func TestObjectStorable(t *testing.T)        { fstests.TestObjectStorable(t) }
+func TestObjectRemove(t *testing.T)          { fstests.TestObjectRemove(t) }
+func TestObjectPurge(t *testing.T)           { fstests.TestObjectPurge(t) }
+func TestFinalise(t *testing.T)              { fstests.TestFinalise(t) }