fstest/test_all: rework integration tests to improve output

- Make integration tests use a config file - Output individual logs for each test - Make HTML report and open browser - Optionally email and upload results
2018-09-29 14:48:29 +01:00 · 2018-09-29 14:48:29 +01:00 · f97c4c8d9d
commit f97c4c8d9d
parent a3c55462a8
8 changed files with 986 additions and 421 deletions
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@ -123,6 +123,13 @@ but they can be run against any of the remotes.
    cd fs/operations
    go test -v -remote TestDrive:

+If you want to use the integration test framework to run these tests
+all together with an HTML report and test retries then from the
+project root:
+
+    go install github.com/ncw/rclone/fstest/test_all
+    test_all -backend drive
+
 If you want to run all the integration tests against all the remotes,
 then change into the project root and run

@ -343,7 +350,7 @@ Unit tests

 Integration tests

-  * Add your fs to `fstest/test_all/test_all.go`
+  * Add your backend to `fstest/test_all/config.yaml`
  * Make sure integration tests pass with
      * `cd fs/operations`
      * `go test -v -remote TestRemote:`
--- a/5
+++ b/5
@ -51,9 +51,8 @@ version:
 # Full suite of integration tests
 test:	rclone
 	go install github.com/ncw/rclone/fstest/test_all
-	-go test -v -count 1 -timeout 20m $(BUILDTAGS) $(GO_FILES) 2>&1 | tee test.log
-	-test_all github.com/ncw/rclone/fs/operations github.com/ncw/rclone/fs/sync 2>&1 | tee fs/test_all.log
-	@echo "Written logs in test.log and fs/test_all.log"
+	-test_all 2>&1 | tee test_all.log
+	@echo "Written logs in test_all.log"

 # Quick test
 quicktest:
--- a/fstest/test_all/clean.go
+++ b/fstest/test_all/clean.go
@ -0,0 +1,60 @@
+// Clean the left over test files
+
+package main
+
+import (
+	"log"
+	"regexp"
+
+	"github.com/ncw/rclone/fs"
+	"github.com/ncw/rclone/fs/list"
+	"github.com/ncw/rclone/fs/operations"
+)
+
+// MatchTestRemote matches the remote names used for testing (copied
+// from fstest/fstest.go so we don't have to import that and get all
+// its flags)
+var MatchTestRemote = regexp.MustCompile(`^rclone-test-[abcdefghijklmnopqrstuvwxyz0123456789]{24}$`)
+
+// cleanFs runs a single clean fs for left over directories
+func cleanFs(remote string) error {
+	f, err := fs.NewFs(remote)
+	if err != nil {
+		return err
+	}
+	entries, err := list.DirSorted(f, true, "")
+	if err != nil {
+		return err
+	}
+	return entries.ForDirError(func(dir fs.Directory) error {
+		dirPath := dir.Remote()
+		fullPath := remote + dirPath
+		if MatchTestRemote.MatchString(dirPath) {
+			if *dryRun {
+				log.Printf("Not Purging %s - -dry-run", fullPath)
+				return nil
+			}
+			log.Printf("Purging %s", fullPath)
+			dir, err := fs.NewFs(fullPath)
+			if err != nil {
+				return err
+			}
+			return operations.Purge(dir, "")
+		}
+		return nil
+	})
+}
+
+// cleanRemotes cleans the list of remotes passed in
+func cleanRemotes(remotes []string) error {
+	var lastError error
+	for _, remote := range remotes {
+		log.Printf("%q - Cleaning", remote)
+		err := cleanFs(remote)
+		if err != nil {
+			lastError = err
+			log.Printf("Failed to purge %q: %v", remote, err)
+		}
+	}
+	return lastError
+}
--- a/fstest/test_all/config.go
+++ b/fstest/test_all/config.go
@ -0,0 +1,159 @@
+// Config handling
+
+package main
+
+import (
+	"io/ioutil"
+	"log"
+	"path"
+
+	"github.com/pkg/errors"
+	"gopkg.in/yaml.v2"
+)
+
+// Test describes an integration test to run with `go test`
+type Test struct {
+	Path       string // path to the source directory
+	SubDir     bool   // if it is possible to add -sub-dir to tests
+	FastList   bool   // if it is possible to add -fast-list to tests
+	AddBackend bool   // set if Path needs the current backend appending
+	NoRetries  bool   // set if no retries should be performed
+}
+
+// Backend describes a backend test
+//
+// FIXME make bucket based remotes set sub-dir automatically???
+type Backend struct {
+	Backend  string // name of the backend directory
+	Remote   string // name of the test remote
+	SubDir   bool   // set to test with -sub-dir
+	FastList bool   // set to test with -fast-list
+}
+
+// MakeRuns creates Run objects the Backend and Test
+//
+// There can be several created, one for each combination of SubDir
+// and FastList
+func (b *Backend) MakeRuns(t *Test) (runs []*Run) {
+	subdirs := []bool{false}
+	if b.SubDir && t.SubDir {
+		subdirs = append(subdirs, true)
+	}
+	fastlists := []bool{false}
+	if b.FastList && t.FastList {
+		fastlists = append(fastlists, true)
+	}
+	for _, subdir := range subdirs {
+		for _, fastlist := range fastlists {
+			run := &Run{
+				Remote:    b.Remote,
+				Backend:   b.Backend,
+				Path:      t.Path,
+				SubDir:    subdir,
+				FastList:  fastlist,
+				NoRetries: t.NoRetries,
+			}
+			if t.AddBackend {
+				run.Path = path.Join(run.Path, b.Backend)
+			}
+			runs = append(runs, run)
+		}
+	}
+	return runs
+}
+
+// Config describes the config for this program
+type Config struct {
+	Tests    []Test
+	Backends []Backend
+}
+
+// NewConfig reads the config file
+func NewConfig(configFile string) (*Config, error) {
+	d, err := ioutil.ReadFile(configFile)
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to read config file")
+	}
+	config := &Config{}
+	err = yaml.Unmarshal(d, &config)
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to parse config file")
+	}
+	// d, err = yaml.Marshal(&config)
+	// if err != nil {
+	// 	log.Fatalf("error: %v", err)
+	// }
+	// fmt.Printf("--- m dump:\n%s\n\n", string(d))
+	return config, nil
+}
+
+// MakeRuns makes Run objects for each combination of Backend and Test
+// in the config
+func (c *Config) MakeRuns() (runs []*Run) {
+	for _, backend := range c.Backends {
+		for _, test := range c.Tests {
+			runs = append(runs, backend.MakeRuns(&test)...)
+		}
+	}
+	return runs
+}
+
+// Filter the Backends with the remotes passed in.
+//
+// If no backend is found with a remote is found then synthesize one
+func (c *Config) filterBackendsByRemotes(remotes []string) {
+	var newBackends []Backend
+	for _, name := range remotes {
+		found := false
+		for i := range c.Backends {
+			if c.Backends[i].Remote == name {
+				newBackends = append(newBackends, c.Backends[i])
+				found = true
+			}
+		}
+		if !found {
+			log.Printf("Remote %q not found - inserting with default flags", name)
+			newBackends = append(newBackends, Backend{Remote: name})
+		}
+	}
+	c.Backends = newBackends
+}
+
+// Filter the Backends with the backendNames passed in
+func (c *Config) filterBackendsByBackends(backendNames []string) {
+	var newBackends []Backend
+	for _, name := range backendNames {
+		for i := range c.Backends {
+			if c.Backends[i].Backend == name {
+				newBackends = append(newBackends, c.Backends[i])
+			}
+		}
+	}
+	c.Backends = newBackends
+}
+
+// Filter the incoming tests into the backends selected
+func (c *Config) filterTests(paths []string) {
+	var newTests []Test
+	for _, path := range paths {
+		for i := range c.Tests {
+			if c.Tests[i].Path == path {
+				newTests = append(newTests, c.Tests[i])
+			}
+		}
+	}
+	c.Tests = newTests
+}
+
+// Remotes returns the unique remotes
+func (c *Config) Remotes() (remotes []string) {
+	found := map[string]struct{}{}
+	for _, backend := range c.Backends {
+		if _, ok := found[backend.Remote]; ok {
+			continue
+		}
+		remotes = append(remotes, backend.Remote)
+		found[backend.Remote] = struct{}{}
+	}
+	return remotes
+}
--- a/fstest/test_all/config.yaml
+++ b/fstest/test_all/config.yaml
@ -0,0 +1,107 @@
+tests:
+ - path: backend
+   addbackend:   true
+   noretries:    true
+ - path: fs/operations
+   subdir:   true
+   fastlist: true
+ - path: fs/sync
+   subdir:   true
+   fastlist: true
+backends:
+ # - backend:  "amazonclouddrive"
+ #   remote:   "TestAmazonCloudDrive:"
+ #   subdir:   false
+ #   fastlist: false
+ - backend:  "b2"
+   remote:   "TestB2:"
+   subdir:   true
+   fastlist: true
+ - backend:  "crypt"
+   remote:   "TestCryptDrive:"
+   subdir:   false
+   fastlist: true
+ - backend:  "crypt"
+   remote:   "TestCryptSwift:"
+   subdir:   false
+   fastlist: false
+ - backend:  "drive"
+   remote:   "TestDrive:"
+   subdir:   false
+   fastlist: true
+ - backend:  "dropbox"
+   remote:   "TestDropbox:"
+   subdir:   false
+   fastlist: false
+ - backend:  "googlecloudstorage"
+   remote:   "TestGoogleCloudStorage:"
+   subdir:   true
+   fastlist: true
+ - backend:  "hubic"
+   remote:   "TestHubic:"
+   subdir:   false
+   fastlist: false
+ - backend:  "jottacloud"
+   remote:   "TestJottacloud:"
+   subdir:   false
+   fastlist: true
+ - backend:  "onedrive"
+   remote:   "TestOneDrive:"
+   subdir:   false
+   fastlist: false
+ - backend:  "s3"
+   remote:   "TestS3:"
+   subdir:   true
+   fastlist: true
+ - backend:  "sftp"
+   remote:   "TestSftp:"
+   subdir:   false
+   fastlist: false
+ - backend:  "swift"
+   remote:   "TestSwift:"
+   subdir:   true
+   fastlist: true
+ - backend:  "yandex"
+   remote:   "TestYandex:"
+   subdir:   false
+   fastlist: false
+ - backend:  "ftp"
+   remote:   "TestFTP:"
+   subdir:   false
+   fastlist: false
+ - backend:  "box"
+   remote:   "TestBox:"
+   subdir:   false
+   fastlist: false
+ - backend:  "qingstor"
+   remote:   "TestQingStor:"
+   subdir:   false
+   fastlist: false
+ - backend:  "azureblob"
+   remote:   "TestAzureBlob:"
+   subdir:   true
+   fastlist: true
+ - backend:  "pcloud"
+   remote:   "TestPcloud:"
+   subdir:   false
+   fastlist: false
+ - backend:  "webdav"
+   remote:   "TestWebdav:"
+   subdir:   false
+   fastlist: false
+ - backend:  "cache"
+   remote:   "TestCache:"
+   subdir:   false
+   fastlist: false
+ - backend:  "mega"
+   remote:   "TestMega:"
+   subdir:   false
+   fastlist: false
+ - backend:  "opendrive"
+   remote:   "TestOpenDrive:"
+   subdir:   false
+   fastlist: false
+ - backend:  "union"
+   remote:   "TestUnion:"
+   subdir:   false
+   fastlist: false
--- a/fstest/test_all/report.go
+++ b/fstest/test_all/report.go
@ -0,0 +1,260 @@
+package main
+
+import (
+	"fmt"
+	"html/template"
+	"io/ioutil"
+	"log"
+	"os"
+	"os/exec"
+	"path"
+	"sort"
+	"time"
+
+	"github.com/ncw/rclone/fs"
+	"github.com/skratchdot/open-golang/open"
+)
+
+const timeFormat = "2006-01-02-150405"
+
+// Report holds the info to make a report on a series of test runs
+type Report struct {
+	LogDir    string        // output directory for logs and report
+	StartTime time.Time     // time started
+	DateTime  string        // directory name for output
+	Duration  time.Duration // time the run took
+	Failed    Runs          // failed runs
+	Passed    Runs          // passed runs
+	Runs      []ReportRun   // runs to report
+	Version   string        // rclone version
+	Previous  string        // previous test name if known
+	IndexHTML string        // path to the index.html file
+	URL       string        // online version
+}
+
+// ReportRun is used in the templates to report on a test run
+type ReportRun struct {
+	Name string
+	Runs Runs
+}
+
+// NewReport initialises and returns a Report
+func NewReport() *Report {
+	r := &Report{
+		StartTime: time.Now(),
+		Version:   fs.Version,
+	}
+	r.DateTime = r.StartTime.Format(timeFormat)
+
+	// Find previous log directory if possible
+	names, err := ioutil.ReadDir(*outputDir)
+	if err == nil && len(names) > 0 {
+		r.Previous = names[len(names)-1].Name()
+	}
+
+	// Create output directory for logs and report
+	r.LogDir = path.Join(*outputDir, r.DateTime)
+	err = os.MkdirAll(r.LogDir, 0777)
+	if err != nil {
+		log.Fatalf("Failed to make log directory: %v", err)
+	}
+
+	// Online version
+	r.URL = *urlBase + r.DateTime + "/index.html"
+
+	return r
+}
+
+// End should be called when the tests are complete
+func (r *Report) End() {
+	r.Duration = time.Since(r.StartTime)
+	sort.Sort(r.Failed)
+	sort.Sort(r.Passed)
+	r.Runs = []ReportRun{
+		{Name: "Failed", Runs: r.Failed},
+		{Name: "Passed", Runs: r.Passed},
+	}
+}
+
+// AllPassed returns true if there were no failed tests
+func (r *Report) AllPassed() bool {
+	return len(r.Failed) == 0
+}
+
+// RecordResult should be called with a Run when it has finished to be
+// recorded into the Report
+func (r *Report) RecordResult(t *Run) {
+	if !t.passed() {
+		r.Failed = append(r.Failed, t)
+	} else {
+		r.Passed = append(r.Passed, t)
+	}
+}
+
+// Title returns a human readable summary title for the Report
+func (r *Report) Title() string {
+	if r.AllPassed() {
+		return fmt.Sprintf("PASS: All tests finished OK in %v", r.Duration)
+	}
+	return fmt.Sprintf("FAIL: %d tests failed in %v", len(r.Failed), r.Duration)
+}
+
+// LogSummary writes the summary to the log file
+func (r *Report) LogSummary() {
+	log.Printf("Logs in %q", r.LogDir)
+
+	// Summarise results
+	log.Printf("SUMMARY")
+	log.Println(r.Title())
+	if !r.AllPassed() {
+		for _, t := range r.Failed {
+			log.Printf("  * %s", toShell(t.nextCmdLine()))
+			log.Printf("    * Failed tests: %v", t.failedTests)
+		}
+	}
+}
+
+// LogHTML writes the summary to index.html in LogDir
+func (r *Report) LogHTML() {
+	r.IndexHTML = path.Join(r.LogDir, "index.html")
+	out, err := os.Create(r.IndexHTML)
+	if err != nil {
+		log.Fatalf("Failed to open index.html: %v", err)
+	}
+	defer func() {
+		err := out.Close()
+		if err != nil {
+			log.Fatalf("Failed to close index.html: %v", err)
+		}
+	}()
+	err = reportTemplate.Execute(out, r)
+	if err != nil {
+		log.Fatalf("Failed to execute template: %v", err)
+	}
+	_ = open.Start("file://" + r.IndexHTML)
+}
+
+var reportHTML = `<!DOCTYPE html>
+<html lang="en">
+<head>
+<meta charset="utf-8">
+<title>{{ .Title }}</title>
+<style>
+table {
+	border-collapse: collapse;
+	border-spacing: 0;
+	border: 1px solid #ddd;
+}
+table.tests {
+	width: 100%;
+}
+table, th, td {
+	border: 1px solid black;
+}
+.Failed {
+	color: red;
+}
+.Passed {
+	color: green;
+}
+.false {
+	font-weight: lighter;
+}
+.true {
+	font-weight: bold;
+}
+
+th, td {
+	text-align: left;
+	padding: 4px;
+}
+
+tr:nth-child(even) {
+    background-color: #f2f2f2
+}
+</style>
+</head>
+<body>
+<h1>{{ .Title }}</h1>
+
+<table>
+<tr><th>Version</th><td>{{ .Version }}</td></tr>
+<tr><th>Date</th><td>{{ .DateTime}} [<a href="{{ .URL }}">online</a>]</td></tr>
+<tr><th>Duration</th><td>{{ .Duration }}</td></tr>
+{{ if .Previous}}<tr><th>Previous</th><td><a href="../{{ .Previous }}/index.html">{{ .Previous }}</a></td></tr>{{ end }}
+<tr><th>Up</th><td><a href="../">Older Tests</a></td></tr>
+</table>
+
+{{ range .Runs }}
+{{ if .Runs }}
+<h2 class="{{ .Name }}">{{ .Name }}: {{ len .Runs }}</h2>
+<table class="{{ .Name }} tests">
+<tr>
+<th>Backend</th>
+<th>Remote</th>
+<th>Test</th>
+<th>SubDir</th>
+<th>FastList</th>
+<th>Failed</th>
+<th>Logs</th>
+</tr>
+{{ $prevBackend := "" }}
+{{ $prevRemote := "" }}
+{{ range .Runs}}
+<tr>
+<td>{{ if ne $prevBackend .Backend }}{{ .Backend }}{{ end }}{{ $prevBackend = .Backend }}</td>
+<td>{{ if ne $prevRemote .Remote }}{{ .Remote }}{{ end }}{{ $prevRemote = .Remote }}</td>
+<td>{{ .Path }}</td>
+<td><span class="{{ .SubDir }}">{{ .SubDir }}</span></td>
+<td><span class="{{ .FastList }}">{{ .FastList }}</span></td>
+<td>{{ .FailedTests }}</td>
+<td>{{ range $i, $v := .Logs }}<a href="{{ $v }}">#{{ $i }}</a> {{ end }}</td>
+</tr>
+{{ end }}
+</table>
+{{ end }}
+{{ end }}
+</body>
+</html>
+`
+
+var reportTemplate = template.Must(template.New("Report").Parse(reportHTML))
+
+// EmailHTML sends the summary report to the email address supplied
+func (r *Report) EmailHTML() {
+	if *emailReport == "" || r.IndexHTML == "" {
+		return
+	}
+	log.Printf("Sending email summary to %q", *emailReport)
+	cmdLine := []string{"mail", "-a", "Content-Type: text/html", *emailReport, "-s", "rclone integration tests: " + r.Title()}
+	cmd := exec.Command(cmdLine[0], cmdLine[1:]...)
+	in, err := os.Open(r.IndexHTML)
+	if err != nil {
+		log.Fatalf("Failed to open index.html: %v", err)
+	}
+	cmd.Stdin = in
+	cmd.Stdout = os.Stdout
+	cmd.Stderr = os.Stderr
+	err = cmd.Run()
+	if err != nil {
+		log.Fatalf("Failed to send email: %v", err)
+	}
+	_ = in.Close()
+}
+
+// Upload uploads a copy of the report online
+func (r *Report) Upload() {
+	if *uploadPath == "" || r.IndexHTML == "" {
+		return
+	}
+	dst := path.Join(*uploadPath, r.DateTime)
+	log.Printf("Uploading results to %q", dst)
+	cmdLine := []string{"rclone", "copy", "-v", r.LogDir, dst}
+	cmd := exec.Command(cmdLine[0], cmdLine[1:]...)
+	cmd.Stdout = os.Stdout
+	cmd.Stderr = os.Stderr
+	err := cmd.Run()
+	if err != nil {
+		log.Fatalf("Failed to upload results: %v", err)
+	}
+}
--- a/fstest/test_all/run.go
+++ b/fstest/test_all/run.go
@ -0,0 +1,318 @@
+// Run a test
+
+package main
+
+import (
+	"bytes"
+	"fmt"
+	"go/build"
+	"io"
+	"log"
+	"os"
+	"os/exec"
+	"path"
+	"regexp"
+	"runtime"
+	"strings"
+	"time"
+
+	"github.com/ncw/rclone/fs"
+)
+
+const testBase = "github.com/ncw/rclone/"
+
+// Run holds info about a running test
+//
+// A run just runs one command line, but it can be run multiple times
+// if retries are needed.
+type Run struct {
+	// Config
+	Remote    string // name of the test remote
+	Backend   string // name of the backend
+	Path      string // path to the source directory
+	SubDir    bool   // add -sub-dir to tests
+	FastList  bool   // add -fast-list to tests
+	NoRetries bool   // don't retry if set
+	// Internals
+	cmdLine     []string
+	cmdString   string
+	try         int
+	err         error
+	output      []byte
+	failedTests []string
+	runFlag     string
+	logDir      string   // directory to place the logs
+	trialName   string   // name/log file name of current trial
+	trialNames  []string // list of all the trials
+}
+
+// Runs records multiple Run objects
+type Runs []*Run
+
+// Sort interface
+func (rs Runs) Len() int      { return len(rs) }
+func (rs Runs) Swap(i, j int) { rs[i], rs[j] = rs[j], rs[i] }
+func (rs Runs) Less(i, j int) bool {
+	a, b := rs[i], rs[j]
+	if a.Backend < b.Backend {
+		return true
+	} else if a.Backend > b.Backend {
+		return false
+	}
+	if a.Remote < b.Remote {
+		return true
+	} else if a.Remote > b.Remote {
+		return false
+	}
+	if a.Path < b.Path {
+		return true
+	} else if a.Path > b.Path {
+		return false
+	}
+	if !a.SubDir && b.SubDir {
+		return true
+	} else if a.SubDir && !b.SubDir {
+		return false
+	}
+	if !a.FastList && b.FastList {
+		return true
+	} else if a.FastList && !b.FastList {
+		return false
+	}
+	return false
+}
+
+// dumpOutput prints the error output
+func (r *Run) dumpOutput() {
+	log.Println("------------------------------------------------------------")
+	log.Printf("---- %q ----", r.cmdString)
+	log.Println(string(r.output))
+	log.Println("------------------------------------------------------------")
+}
+
+var failRe = regexp.MustCompile(`(?m)^--- FAIL: (Test\w*) \(`)
+
+// findFailures looks for all the tests which failed
+func (r *Run) findFailures() {
+	oldFailedTests := r.failedTests
+	r.failedTests = nil
+	for _, matches := range failRe.FindAllSubmatch(r.output, -1) {
+		r.failedTests = append(r.failedTests, string(matches[1]))
+	}
+	if len(r.failedTests) != 0 {
+		r.runFlag = "^(" + strings.Join(r.failedTests, "|") + ")$"
+	} else {
+		r.runFlag = ""
+	}
+	if r.passed() && len(r.failedTests) != 0 {
+		log.Printf("%q - Expecting no errors but got: %v", r.cmdString, r.failedTests)
+		r.dumpOutput()
+	} else if !r.passed() && len(r.failedTests) == 0 {
+		log.Printf("%q - Expecting errors but got none: %v", r.cmdString, r.failedTests)
+		r.dumpOutput()
+		r.failedTests = oldFailedTests
+	}
+}
+
+// nextCmdLine returns the next command line
+func (r *Run) nextCmdLine() []string {
+	cmdLine := r.cmdLine
+	if r.runFlag != "" {
+		cmdLine = append(cmdLine, "-test.run", r.runFlag)
+	}
+	return cmdLine
+}
+
+// trial runs a single test
+func (r *Run) trial() {
+	cmdLine := r.nextCmdLine()
+	cmdString := toShell(cmdLine)
+	msg := fmt.Sprintf("%q - Starting (try %d/%d)", cmdString, r.try, *maxTries)
+	log.Println(msg)
+	logName := path.Join(r.logDir, r.trialName)
+	out, err := os.Create(logName)
+	if err != nil {
+		log.Fatalf("Couldn't create log file: %v", err)
+	}
+	defer func() {
+		err := out.Close()
+		if err != nil {
+			log.Fatalf("Failed to close log file: %v", err)
+		}
+	}()
+	_, _ = fmt.Fprintln(out, msg)
+
+	// Early exit if --try-run
+	if *dryRun {
+		log.Printf("Not executing as --dry-run: %v", cmdLine)
+		_, _ = fmt.Fprintln(out, "--dry-run is set - not running")
+		return
+	}
+
+	// Internal buffer
+	var b bytes.Buffer
+	multiOut := io.MultiWriter(out, &b)
+
+	cmd := exec.Command(cmdLine[0], cmdLine[1:]...)
+	cmd.Stderr = multiOut
+	cmd.Stdout = multiOut
+	start := time.Now()
+	r.err = cmd.Run()
+	r.output = b.Bytes()
+	duration := time.Since(start)
+	r.findFailures()
+	if r.passed() {
+		msg = fmt.Sprintf("%q - Finished OK in %v (try %d/%d)", cmdString, duration, r.try, *maxTries)
+	} else {
+		msg = fmt.Sprintf("%q - Finished ERROR in %v (try %d/%d): %v: Failed %v", cmdString, duration, r.try, *maxTries, r.err, r.failedTests)
+	}
+	log.Println(msg)
+	_, _ = fmt.Fprintln(out, msg)
+}
+
+// passed returns true if the test passed
+func (r *Run) passed() bool {
+	return r.err == nil
+}
+
+// GOPATH returns the current GOPATH
+func GOPATH() string {
+	gopath := os.Getenv("GOPATH")
+	if gopath == "" {
+		gopath = build.Default.GOPATH
+	}
+	return gopath
+}
+
+// BinaryName turns a package name into a binary name
+func (r *Run) BinaryName() string {
+	binary := path.Base(r.Path) + ".test"
+	if runtime.GOOS == "windows" {
+		binary += ".exe"
+	}
+	return binary
+}
+
+// BinaryPath turns a package name into a binary path
+func (r *Run) BinaryPath() string {
+	return path.Join(r.Path, r.BinaryName())
+}
+
+// PackagePath returns the path to the package
+func (r *Run) PackagePath() string {
+	return path.Join(GOPATH(), "src", r.Path)
+}
+
+// Chdir into the package directory
+func (r *Run) Chdir() {
+	err := os.Chdir(r.PackagePath())
+	if err != nil {
+		log.Fatalf("Failed to chdir to package %q: %v", r.Path, err)
+	}
+}
+
+// MakeTestBinary makes the binary we will run
+func (r *Run) MakeTestBinary() {
+	binary := r.BinaryPath()
+	binaryName := r.BinaryName()
+	log.Printf("%s: Making test binary %q", r.Path, binaryName)
+	cmdLine := []string{"go", "test", "-c", "-o", binary, testBase + r.Path}
+	if *dryRun {
+		log.Printf("Not executing: %v", cmdLine)
+		return
+	}
+	err := exec.Command(cmdLine[0], cmdLine[1:]...).Run()
+	if err != nil {
+		log.Fatalf("Failed to make test binary: %v", err)
+	}
+	if _, err := os.Stat(binary); err != nil {
+		log.Fatalf("Couldn't find test binary %q", binary)
+	}
+}
+
+// RemoveTestBinary removes the binary made in makeTestBinary
+func (r *Run) RemoveTestBinary() {
+	if *dryRun {
+		return
+	}
+	binary := r.BinaryPath()
+	err := os.Remove(binary) // Delete the binary when finished
+	if err != nil {
+		log.Printf("Error removing test binary %q: %v", binary, err)
+	}
+}
+
+// Name returns the run name as a file name friendly string
+func (r *Run) Name() string {
+	ns := []string{
+		r.Backend,
+		strings.Replace(r.Path, "/", ".", -1),
+		r.Remote,
+	}
+	if r.SubDir {
+		ns = append(ns, "subdir")
+	}
+	if r.FastList {
+		ns = append(ns, "fastlist")
+	}
+	ns = append(ns, fmt.Sprintf("%d", r.try))
+	s := strings.Join(ns, "-")
+	s = strings.Replace(s, ":", "", -1)
+	return s
+}
+
+// Init the Run
+func (r *Run) Init() {
+	binary := r.BinaryPath()
+	r.cmdLine = []string{binary, "-test.v", "-test.timeout", timeout.String(), "-remote", r.Remote}
+	r.try = 1
+	if *verbose {
+		r.cmdLine = append(r.cmdLine, "-verbose")
+		fs.Config.LogLevel = fs.LogLevelDebug
+	}
+	if *runOnly != "" {
+		r.cmdLine = append(r.cmdLine, "-test.run", *runOnly)
+	}
+	if r.SubDir {
+		r.cmdLine = append(r.cmdLine, "-subdir")
+	}
+	if r.FastList {
+		r.cmdLine = append(r.cmdLine, "-fast-list")
+	}
+	r.cmdString = toShell(r.cmdLine)
+}
+
+// Logs returns all the log names
+func (r *Run) Logs() []string {
+	return r.trialNames
+}
+
+// FailedTests returns the failed tests as a comma separated string, limiting the number
+func (r *Run) FailedTests() string {
+	const maxTests = 5
+	ts := r.failedTests
+	if len(ts) > maxTests {
+		ts = ts[:maxTests:maxTests]
+		ts = append(ts, fmt.Sprintf("… (%d more)", len(r.failedTests)-maxTests))
+	}
+	return strings.Join(ts, ", ")
+}
+
+// Run runs all the trials for this test
+func (r *Run) Run(logDir string, result chan<- *Run) {
+	r.Init()
+	r.logDir = logDir
+	for r.try = 1; r.try <= *maxTries; r.try++ {
+		r.trialName = r.Name() + ".txt"
+		r.trialNames = append(r.trialNames, r.trialName)
+		log.Printf("Starting run with log %q", r.trialName)
+		r.trial()
+		if r.passed() || r.NoRetries {
+			break
+		}
+	}
+	if !r.passed() {
+		r.dumpOutput()
+	}
+	result <- r
+}
--- a/fstest/test_all/test_all.go
+++ b/fstest/test_all/test_all.go
@ -4,24 +4,22 @@
 // See the `test` target in the Makefile.
 package main

+/* FIXME
+
+Make TesTrun have a []string of flags to try - that then makes it generic
+
+*/
+
 import (
 	"flag"
-	"go/build"
 	"log"
 	"os"
-	"os/exec"
 	"path"
 	"regexp"
-	"runtime"
 	"strings"
 	"time"

 	_ "github.com/ncw/rclone/backend/all" // import all fs
-	"github.com/ncw/rclone/fs"
-	"github.com/ncw/rclone/fs/config"
-	"github.com/ncw/rclone/fs/list"
-	"github.com/ncw/rclone/fs/operations"
-	"github.com/ncw/rclone/fstest"
 )

 type remoteConfig struct {
@ -31,218 +29,23 @@ type remoteConfig struct {
 }

 var (
-	remotes = []remoteConfig{
-		// {
-		// 	Name:     "TestAmazonCloudDrive:",
-		// 	SubDir:   false,
-		// 	FastList: false,
-		// },
-		{
-			Name:     "TestB2:",
-			SubDir:   true,
-			FastList: true,
-		},
-		{
-			Name:     "TestCryptDrive:",
-			SubDir:   false,
-			FastList: true,
-		},
-		{
-			Name:     "TestCryptSwift:",
-			SubDir:   false,
-			FastList: false,
-		},
-		{
-			Name:     "TestDrive:",
-			SubDir:   false,
-			FastList: true,
-		},
-		{
-			Name:     "TestDropbox:",
-			SubDir:   false,
-			FastList: false,
-		},
-		{
-			Name:     "TestGoogleCloudStorage:",
-			SubDir:   true,
-			FastList: true,
-		},
-		{
-			Name:     "TestHubic:",
-			SubDir:   false,
-			FastList: false,
-		},
-		{
-			Name:     "TestJottacloud:",
-			SubDir:   false,
-			FastList: true,
-		},
-		{
-			Name:     "TestOneDrive:",
-			SubDir:   false,
-			FastList: false,
-		},
-		{
-			Name:     "TestS3:",
-			SubDir:   true,
-			FastList: true,
-		},
-		{
-			Name:     "TestSftp:",
-			SubDir:   false,
-			FastList: false,
-		},
-		{
-			Name:     "TestSwift:",
-			SubDir:   true,
-			FastList: true,
-		},
-		{
-			Name:     "TestYandex:",
-			SubDir:   false,
-			FastList: false,
-		},
-		{
-			Name:     "TestFTP:",
-			SubDir:   false,
-			FastList: false,
-		},
-		{
-			Name:     "TestBox:",
-			SubDir:   false,
-			FastList: false,
-		},
-		{
-			Name:     "TestQingStor:",
-			SubDir:   false,
-			FastList: false,
-		},
-		{
-			Name:     "TestAzureBlob:",
-			SubDir:   true,
-			FastList: true,
-		},
-		{
-			Name:     "TestPcloud:",
-			SubDir:   false,
-			FastList: false,
-		},
-		{
-			Name:     "TestWebdav:",
-			SubDir:   false,
-			FastList: false,
-		},
-		{
-			Name:     "TestCache:",
-			SubDir:   false,
-			FastList: false,
-		},
-		{
-			Name:     "TestMega:",
-			SubDir:   false,
-			FastList: false,
-		},
-		{
-			Name:     "TestOpenDrive:",
-			SubDir:   false,
-			FastList: false,
-		},
-		{
-			Name:     "TestUnion:",
-			SubDir:   false,
-			FastList: false,
-		},
-	}
 	// Flags
-	maxTries = flag.Int("maxtries", 5, "Number of times to try each test")
-	runTests = flag.String("remotes", "", "Comma separated list of remotes to test, eg 'TestSwift:,TestS3'")
-	clean    = flag.Bool("clean", false, "Instead of testing, clean all left over test directories")
-	runOnly  = flag.String("run", "", "Run only those tests matching the regexp supplied")
-	timeout  = flag.Duration("timeout", 30*time.Minute, "Maximum time to run each test for before giving up")
+	maxTries     = flag.Int("maxtries", 5, "Number of times to try each test")
+	testRemotes  = flag.String("remotes", "", "Comma separated list of remotes to test, eg 'TestSwift:,TestS3'")
+	testBackends = flag.String("backends", "", "Comma separated list of backends to test, eg 's3,googlecloudstorage")
+	testTests    = flag.String("tests", "", "Comma separated list of tests to test, eg 'fs/sync,fs/operations'")
+	clean        = flag.Bool("clean", false, "Instead of testing, clean all left over test directories")
+	runOnly      = flag.String("run", "", "Run only those tests matching the regexp supplied")
+	timeout      = flag.Duration("timeout", 30*time.Minute, "Maximum time to run each test for before giving up")
+	configFile   = flag.String("config", "fstest/test_all/config.yaml", "Path to config file")
+	outputDir    = flag.String("output", path.Join(os.TempDir(), "rclone-integration-tests"), "Place to store results")
+	emailReport  = flag.String("email", "", "Set to email the report to the address supplied")
+	dryRun       = flag.Bool("dry-run", false, "Print commands which would be executed only")
+	urlBase      = flag.String("url-base", "https://pub.rclone.org/integration-tests/", "Base for the online version")
+	uploadPath   = flag.String("upload", "", "Set this to an rclone path to upload the results here")
+	verbose      = flag.Bool("verbose", false, "Set to enable verbose logging in the tests")
 )

-// test holds info about a running test
-type test struct {
-	pkg         string
-	remote      string
-	subdir      bool
-	cmdLine     []string
-	cmdString   string
-	try         int
-	err         error
-	output      []byte
-	failedTests []string
-	runFlag     string
-}
-
-// newTest creates a new test
-func newTest(pkg, remote string, subdir bool, fastlist bool) *test {
-	binary := pkgBinary(pkg)
-	t := &test{
-		pkg:     pkg,
-		remote:  remote,
-		subdir:  subdir,
-		cmdLine: []string{binary, "-test.timeout", timeout.String(), "-remote", remote},
-		try:     1,
-	}
-	if *fstest.Verbose {
-		t.cmdLine = append(t.cmdLine, "-test.v")
-		fs.Config.LogLevel = fs.LogLevelDebug
-	}
-	if *runOnly != "" {
-		t.cmdLine = append(t.cmdLine, "-test.run", *runOnly)
-	}
-	if subdir {
-		t.cmdLine = append(t.cmdLine, "-subdir")
-	}
-	if fastlist {
-		t.cmdLine = append(t.cmdLine, "-fast-list")
-	}
-	t.cmdString = toShell(t.cmdLine)
-	return t
-}
-
-// dumpOutput prints the error output
-func (t *test) dumpOutput() {
-	log.Println("------------------------------------------------------------")
-	log.Printf("---- %q ----", t.cmdString)
-	log.Println(string(t.output))
-	log.Println("------------------------------------------------------------")
-}
-
-var failRe = regexp.MustCompile(`(?m)^--- FAIL: (Test\w*) \(`)
-
-// findFailures looks for all the tests which failed
-func (t *test) findFailures() {
-	oldFailedTests := t.failedTests
-	t.failedTests = nil
-	for _, matches := range failRe.FindAllSubmatch(t.output, -1) {
-		t.failedTests = append(t.failedTests, string(matches[1]))
-	}
-	if len(t.failedTests) != 0 {
-		t.runFlag = "^(" + strings.Join(t.failedTests, "|") + ")$"
-	} else {
-		t.runFlag = ""
-	}
-	if t.passed() && len(t.failedTests) != 0 {
-		log.Printf("%q - Expecting no errors but got: %v", t.cmdString, t.failedTests)
-		t.dumpOutput()
-	} else if !t.passed() && len(t.failedTests) == 0 {
-		log.Printf("%q - Expecting errors but got none: %v", t.cmdString, t.failedTests)
-		t.dumpOutput()
-		t.failedTests = oldFailedTests
-	}
-}
-
-// nextCmdLine returns the next command line
-func (t *test) nextCmdLine() []string {
-	cmdLine := t.cmdLine
-	if t.runFlag != "" {
-		cmdLine = append(cmdLine, "-test.run", t.runFlag)
-	}
-	return cmdLine
-}
-
 // if matches then is definitely OK in the shell
 var shellOK = regexp.MustCompile("^[A-Za-z0-9./_:-]+$")

@ -261,181 +64,53 @@ func toShell(args []string) (result string) {
 	return result
 }

-// trial runs a single test
-func (t *test) trial() {
-	cmdLine := t.nextCmdLine()
-	cmdString := toShell(cmdLine)
-	log.Printf("%q - Starting (try %d/%d)", cmdString, t.try, *maxTries)
-	cmd := exec.Command(cmdLine[0], cmdLine[1:]...)
-	start := time.Now()
-	t.output, t.err = cmd.CombinedOutput()
-	duration := time.Since(start)
-	t.findFailures()
-	if t.passed() {
-		log.Printf("%q - Finished OK in %v (try %d/%d)", cmdString, duration, t.try, *maxTries)
-	} else {
-		log.Printf("%q - Finished ERROR in %v (try %d/%d): %v: Failed %v", cmdString, duration, t.try, *maxTries, t.err, t.failedTests)
-	}
-}
-
-// cleanFs runs a single clean fs for left over directories
-func (t *test) cleanFs() error {
-	f, err := fs.NewFs(t.remote)
-	if err != nil {
-		return err
-	}
-	entries, err := list.DirSorted(f, true, "")
-	if err != nil {
-		return err
-	}
-	return entries.ForDirError(func(dir fs.Directory) error {
-		remote := dir.Remote()
-		if fstest.MatchTestRemote.MatchString(remote) {
-			log.Printf("Purging %s%s", t.remote, remote)
-			dir, err := fs.NewFs(t.remote + remote)
-			if err != nil {
-				return err
-			}
-			return operations.Purge(dir, "")
-		}
-		return nil
-	})
-}
-
-// clean runs a single clean on a fs for left over directories
-func (t *test) clean() {
-	log.Printf("%q - Starting clean (try %d/%d)", t.remote, t.try, *maxTries)
-	start := time.Now()
-	t.err = t.cleanFs()
-	if t.err != nil {
-		log.Printf("%q - Failed to purge %v", t.remote, t.err)
-	}
-	duration := time.Since(start)
-	if t.passed() {
-		log.Printf("%q - Finished OK in %v (try %d/%d)", t.cmdString, duration, t.try, *maxTries)
-	} else {
-		log.Printf("%q - Finished ERROR in %v (try %d/%d): %v", t.cmdString, duration, t.try, *maxTries, t.err)
-	}
-}
-
-// passed returns true if the test passed
-func (t *test) passed() bool {
-	return t.err == nil
-}
-
-// run runs all the trials for this test
-func (t *test) run(result chan<- *test) {
-	for t.try = 1; t.try <= *maxTries; t.try++ {
-		if *clean {
-			if !t.subdir {
-				t.clean()
-			}
-		} else {
-			t.trial()
-		}
-		if t.passed() {
-			break
-		}
-	}
-	if !t.passed() {
-		t.dumpOutput()
-	}
-	result <- t
-}
-
-// GOPATH returns the current GOPATH
-func GOPATH() string {
-	gopath := os.Getenv("GOPATH")
-	if gopath == "" {
-		gopath = build.Default.GOPATH
-	}
-	return gopath
-}
-
-// turn a package name into a binary name
-func pkgBinaryName(pkg string) string {
-	binary := path.Base(pkg) + ".test"
-	if runtime.GOOS == "windows" {
-		binary += ".exe"
-	}
-	return binary
-}
-
-// turn a package name into a binary path
-func pkgBinary(pkg string) string {
-	return path.Join(pkgPath(pkg), pkgBinaryName(pkg))
-}
-
-// returns the path to the package
-func pkgPath(pkg string) string {
-	return path.Join(GOPATH(), "src", pkg)
-}
-
-// cd into the package directory
-func pkgChdir(pkg string) {
-	err := os.Chdir(pkgPath(pkg))
-	if err != nil {
-		log.Fatalf("Failed to chdir to package %q: %v", pkg, err)
-	}
-}
-
-// makeTestBinary makes the binary we will run
-func makeTestBinary(pkg string) {
-	binaryName := pkgBinaryName(pkg)
-	log.Printf("%s: Making test binary %q", pkg, binaryName)
-	pkgChdir(pkg)
-	err := exec.Command("go", "test", "-c", "-o", binaryName).Run()
-	if err != nil {
-		log.Fatalf("Failed to make test binary: %v", err)
-	}
-	binary := pkgBinary(pkg)
-	if _, err := os.Stat(binary); err != nil {
-		log.Fatalf("Couldn't find test binary %q", binary)
-	}
-}
-
-// removeTestBinary removes the binary made in makeTestBinary
-func removeTestBinary(pkg string) {
-	binary := pkgBinary(pkg)
-	err := os.Remove(binary) // Delete the binary when finished
-	if err != nil {
-		log.Printf("Error removing test binary %q: %v", binary, err)
-	}
-}
-
 func main() {
 	flag.Parse()
-	packages := flag.Args()
-	log.Printf("Testing packages: %s", strings.Join(packages, ", "))
-	if *runTests != "" {
-		newRemotes := []remoteConfig{}
-		for _, name := range strings.Split(*runTests, ",") {
-			for i := range remotes {
-				if remotes[i].Name == name {
-					newRemotes = append(newRemotes, remotes[i])
-					goto found
-				}
-			}
-			log.Printf("Remote %q not found - inserting with default flags", name)
-			newRemotes = append(newRemotes, remoteConfig{Name: name})
-		found:
-		}
-		remotes = newRemotes
+	conf, err := NewConfig(*configFile)
+	if err != nil {
+		log.Println("test_all should be run from the root of the rclone source code")
+		log.Fatal(err)
 	}
+
+	// Filter selection
+	if *testRemotes != "" {
+		conf.filterBackendsByRemotes(strings.Split(*testRemotes, ","))
+	}
+	if *testBackends != "" {
+		conf.filterBackendsByBackends(strings.Split(*testBackends, ","))
+	}
+	if *testTests != "" {
+		conf.filterTests(strings.Split(*testTests, ","))
+	}
+
+	// Just clean the directories if required
+	if *clean {
+		err := cleanRemotes(conf.Remotes())
+		if err != nil {
+			log.Fatalf("Failed to clean: %v", err)
+		}
+		return
+	}
+
 	var names []string
-	for _, remote := range remotes {
-		names = append(names, remote.Name)
+	for _, remote := range conf.Backends {
+		names = append(names, remote.Remote)
 	}
 	log.Printf("Testing remotes: %s", strings.Join(names, ", "))

-	start := time.Now()
-	if *clean {
-		config.LoadConfig()
-		packages = []string{"clean"}
-	} else {
-		for _, pkg := range packages {
-			makeTestBinary(pkg)
-			defer removeTestBinary(pkg)
+	// Runs we will do for this test
+	runs := conf.MakeRuns()
+
+	// Create Report
+	report := NewReport()
+
+	// Make the test binaries, one per Path found in the tests
+	done := map[string]struct{}{}
+	for _, run := range runs {
+		if _, found := done[run.Path]; !found {
+			done[run.Path] = struct{}{}
+			run.MakeTestBinary()
+			defer run.RemoveTestBinary()
 		}
 	}

@ -443,46 +118,26 @@ func main() {
 	_ = os.Setenv("RCLONE_CACHE_DB_WAIT_TIME", "30m")

 	// start the tests
-	results := make(chan *test, 8)
+	results := make(chan *Run, 8)
 	awaiting := 0
-	bools := []bool{false, true}
-	if *clean {
-		// Don't run -subdir and -fast-list if -clean
-		bools = bools[:1]
-	}
-	for _, pkg := range packages {
-		for _, remote := range remotes {
-			for _, subdir := range bools {
-				for _, fastlist := range bools {
-					if (!subdir || subdir && remote.SubDir) && (!fastlist || fastlist && remote.FastList) {
-						go newTest(pkg, remote.Name, subdir, fastlist).run(results)
-						awaiting++
-					}
-				}
-			}
-		}
+	for _, run := range runs {
+		go run.Run(report.LogDir, results)
+		awaiting++
 	}

 	// Wait for the tests to finish
-	var failed []*test
 	for ; awaiting > 0; awaiting-- {
 		t := <-results
-		if !t.passed() {
-			failed = append(failed, t)
-		}
+		report.RecordResult(t)
 	}
-	duration := time.Since(start)

-	// Summarise results
-	log.Printf("SUMMARY")
-	if len(failed) == 0 {
-		log.Printf("PASS: All tests finished OK in %v", duration)
-	} else {
-		log.Printf("FAIL: %d tests failed in %v", len(failed), duration)
-		for _, t := range failed {
-			log.Printf("  * %s", toShell(t.nextCmdLine()))
-			log.Printf("    * Failed tests: %v", t.failedTests)
-		}
+	// Log and exit
+	report.End()
+	report.LogSummary()
+	report.LogHTML()
+	report.EmailHTML()
+	report.Upload()
+	if !report.AllPassed() {
 		os.Exit(1)
 	}
 }