fstest/test_all: rework integration tests to improve output

- Make integration tests use a config file
- Output individual logs for each test
- Make HTML report and open browser
- Optionally email and upload results
This commit is contained in:
Nick Craig-Wood 2018-09-29 14:48:29 +01:00
parent a3c55462a8
commit f97c4c8d9d
8 changed files with 986 additions and 421 deletions

View file

@ -123,6 +123,13 @@ but they can be run against any of the remotes.
cd fs/operations cd fs/operations
go test -v -remote TestDrive: go test -v -remote TestDrive:
If you want to use the integration test framework to run these tests
all together with an HTML report and test retries then from the
project root:
go install github.com/ncw/rclone/fstest/test_all
test_all -backend drive
If you want to run all the integration tests against all the remotes, If you want to run all the integration tests against all the remotes,
then change into the project root and run then change into the project root and run
@ -343,7 +350,7 @@ Unit tests
Integration tests Integration tests
* Add your fs to `fstest/test_all/test_all.go` * Add your backend to `fstest/test_all/config.yaml`
* Make sure integration tests pass with * Make sure integration tests pass with
* `cd fs/operations` * `cd fs/operations`
* `go test -v -remote TestRemote:` * `go test -v -remote TestRemote:`

View file

@ -51,9 +51,8 @@ version:
# Full suite of integration tests # Full suite of integration tests
test: rclone test: rclone
go install github.com/ncw/rclone/fstest/test_all go install github.com/ncw/rclone/fstest/test_all
-go test -v -count 1 -timeout 20m $(BUILDTAGS) $(GO_FILES) 2>&1 | tee test.log -test_all 2>&1 | tee test_all.log
-test_all github.com/ncw/rclone/fs/operations github.com/ncw/rclone/fs/sync 2>&1 | tee fs/test_all.log @echo "Written logs in test_all.log"
@echo "Written logs in test.log and fs/test_all.log"
# Quick test # Quick test
quicktest: quicktest:

60
fstest/test_all/clean.go Normal file
View file

@ -0,0 +1,60 @@
// Clean the left over test files
package main
import (
"log"
"regexp"
"github.com/ncw/rclone/fs"
"github.com/ncw/rclone/fs/list"
"github.com/ncw/rclone/fs/operations"
)
// MatchTestRemote matches the remote names used for testing (copied
// from fstest/fstest.go so we don't have to import that and get all
// its flags)
var MatchTestRemote = regexp.MustCompile(`^rclone-test-[abcdefghijklmnopqrstuvwxyz0123456789]{24}$`)
// cleanFs runs a single clean fs for left over directories
func cleanFs(remote string) error {
f, err := fs.NewFs(remote)
if err != nil {
return err
}
entries, err := list.DirSorted(f, true, "")
if err != nil {
return err
}
return entries.ForDirError(func(dir fs.Directory) error {
dirPath := dir.Remote()
fullPath := remote + dirPath
if MatchTestRemote.MatchString(dirPath) {
if *dryRun {
log.Printf("Not Purging %s - -dry-run", fullPath)
return nil
}
log.Printf("Purging %s", fullPath)
dir, err := fs.NewFs(fullPath)
if err != nil {
return err
}
return operations.Purge(dir, "")
}
return nil
})
}
// cleanRemotes cleans the list of remotes passed in
func cleanRemotes(remotes []string) error {
var lastError error
for _, remote := range remotes {
log.Printf("%q - Cleaning", remote)
err := cleanFs(remote)
if err != nil {
lastError = err
log.Printf("Failed to purge %q: %v", remote, err)
}
}
return lastError
}

159
fstest/test_all/config.go Normal file
View file

@ -0,0 +1,159 @@
// Config handling
package main
import (
"io/ioutil"
"log"
"path"
"github.com/pkg/errors"
"gopkg.in/yaml.v2"
)
// Test describes an integration test to run with `go test`
type Test struct {
Path string // path to the source directory
SubDir bool // if it is possible to add -sub-dir to tests
FastList bool // if it is possible to add -fast-list to tests
AddBackend bool // set if Path needs the current backend appending
NoRetries bool // set if no retries should be performed
}
// Backend describes a backend test
//
// FIXME make bucket based remotes set sub-dir automatically???
type Backend struct {
Backend string // name of the backend directory
Remote string // name of the test remote
SubDir bool // set to test with -sub-dir
FastList bool // set to test with -fast-list
}
// MakeRuns creates Run objects the Backend and Test
//
// There can be several created, one for each combination of SubDir
// and FastList
func (b *Backend) MakeRuns(t *Test) (runs []*Run) {
subdirs := []bool{false}
if b.SubDir && t.SubDir {
subdirs = append(subdirs, true)
}
fastlists := []bool{false}
if b.FastList && t.FastList {
fastlists = append(fastlists, true)
}
for _, subdir := range subdirs {
for _, fastlist := range fastlists {
run := &Run{
Remote: b.Remote,
Backend: b.Backend,
Path: t.Path,
SubDir: subdir,
FastList: fastlist,
NoRetries: t.NoRetries,
}
if t.AddBackend {
run.Path = path.Join(run.Path, b.Backend)
}
runs = append(runs, run)
}
}
return runs
}
// Config describes the config for this program
type Config struct {
Tests []Test
Backends []Backend
}
// NewConfig reads the config file
func NewConfig(configFile string) (*Config, error) {
d, err := ioutil.ReadFile(configFile)
if err != nil {
return nil, errors.Wrap(err, "failed to read config file")
}
config := &Config{}
err = yaml.Unmarshal(d, &config)
if err != nil {
return nil, errors.Wrap(err, "failed to parse config file")
}
// d, err = yaml.Marshal(&config)
// if err != nil {
// log.Fatalf("error: %v", err)
// }
// fmt.Printf("--- m dump:\n%s\n\n", string(d))
return config, nil
}
// MakeRuns makes Run objects for each combination of Backend and Test
// in the config
func (c *Config) MakeRuns() (runs []*Run) {
for _, backend := range c.Backends {
for _, test := range c.Tests {
runs = append(runs, backend.MakeRuns(&test)...)
}
}
return runs
}
// Filter the Backends with the remotes passed in.
//
// If no backend is found with a remote is found then synthesize one
func (c *Config) filterBackendsByRemotes(remotes []string) {
var newBackends []Backend
for _, name := range remotes {
found := false
for i := range c.Backends {
if c.Backends[i].Remote == name {
newBackends = append(newBackends, c.Backends[i])
found = true
}
}
if !found {
log.Printf("Remote %q not found - inserting with default flags", name)
newBackends = append(newBackends, Backend{Remote: name})
}
}
c.Backends = newBackends
}
// Filter the Backends with the backendNames passed in
func (c *Config) filterBackendsByBackends(backendNames []string) {
var newBackends []Backend
for _, name := range backendNames {
for i := range c.Backends {
if c.Backends[i].Backend == name {
newBackends = append(newBackends, c.Backends[i])
}
}
}
c.Backends = newBackends
}
// Filter the incoming tests into the backends selected
func (c *Config) filterTests(paths []string) {
var newTests []Test
for _, path := range paths {
for i := range c.Tests {
if c.Tests[i].Path == path {
newTests = append(newTests, c.Tests[i])
}
}
}
c.Tests = newTests
}
// Remotes returns the unique remotes
func (c *Config) Remotes() (remotes []string) {
found := map[string]struct{}{}
for _, backend := range c.Backends {
if _, ok := found[backend.Remote]; ok {
continue
}
remotes = append(remotes, backend.Remote)
found[backend.Remote] = struct{}{}
}
return remotes
}

107
fstest/test_all/config.yaml Normal file
View file

@ -0,0 +1,107 @@
tests:
- path: backend
addbackend: true
noretries: true
- path: fs/operations
subdir: true
fastlist: true
- path: fs/sync
subdir: true
fastlist: true
backends:
# - backend: "amazonclouddrive"
# remote: "TestAmazonCloudDrive:"
# subdir: false
# fastlist: false
- backend: "b2"
remote: "TestB2:"
subdir: true
fastlist: true
- backend: "crypt"
remote: "TestCryptDrive:"
subdir: false
fastlist: true
- backend: "crypt"
remote: "TestCryptSwift:"
subdir: false
fastlist: false
- backend: "drive"
remote: "TestDrive:"
subdir: false
fastlist: true
- backend: "dropbox"
remote: "TestDropbox:"
subdir: false
fastlist: false
- backend: "googlecloudstorage"
remote: "TestGoogleCloudStorage:"
subdir: true
fastlist: true
- backend: "hubic"
remote: "TestHubic:"
subdir: false
fastlist: false
- backend: "jottacloud"
remote: "TestJottacloud:"
subdir: false
fastlist: true
- backend: "onedrive"
remote: "TestOneDrive:"
subdir: false
fastlist: false
- backend: "s3"
remote: "TestS3:"
subdir: true
fastlist: true
- backend: "sftp"
remote: "TestSftp:"
subdir: false
fastlist: false
- backend: "swift"
remote: "TestSwift:"
subdir: true
fastlist: true
- backend: "yandex"
remote: "TestYandex:"
subdir: false
fastlist: false
- backend: "ftp"
remote: "TestFTP:"
subdir: false
fastlist: false
- backend: "box"
remote: "TestBox:"
subdir: false
fastlist: false
- backend: "qingstor"
remote: "TestQingStor:"
subdir: false
fastlist: false
- backend: "azureblob"
remote: "TestAzureBlob:"
subdir: true
fastlist: true
- backend: "pcloud"
remote: "TestPcloud:"
subdir: false
fastlist: false
- backend: "webdav"
remote: "TestWebdav:"
subdir: false
fastlist: false
- backend: "cache"
remote: "TestCache:"
subdir: false
fastlist: false
- backend: "mega"
remote: "TestMega:"
subdir: false
fastlist: false
- backend: "opendrive"
remote: "TestOpenDrive:"
subdir: false
fastlist: false
- backend: "union"
remote: "TestUnion:"
subdir: false
fastlist: false

260
fstest/test_all/report.go Normal file
View file

@ -0,0 +1,260 @@
package main
import (
"fmt"
"html/template"
"io/ioutil"
"log"
"os"
"os/exec"
"path"
"sort"
"time"
"github.com/ncw/rclone/fs"
"github.com/skratchdot/open-golang/open"
)
const timeFormat = "2006-01-02-150405"
// Report holds the info to make a report on a series of test runs
type Report struct {
LogDir string // output directory for logs and report
StartTime time.Time // time started
DateTime string // directory name for output
Duration time.Duration // time the run took
Failed Runs // failed runs
Passed Runs // passed runs
Runs []ReportRun // runs to report
Version string // rclone version
Previous string // previous test name if known
IndexHTML string // path to the index.html file
URL string // online version
}
// ReportRun is used in the templates to report on a test run
type ReportRun struct {
Name string
Runs Runs
}
// NewReport initialises and returns a Report
func NewReport() *Report {
r := &Report{
StartTime: time.Now(),
Version: fs.Version,
}
r.DateTime = r.StartTime.Format(timeFormat)
// Find previous log directory if possible
names, err := ioutil.ReadDir(*outputDir)
if err == nil && len(names) > 0 {
r.Previous = names[len(names)-1].Name()
}
// Create output directory for logs and report
r.LogDir = path.Join(*outputDir, r.DateTime)
err = os.MkdirAll(r.LogDir, 0777)
if err != nil {
log.Fatalf("Failed to make log directory: %v", err)
}
// Online version
r.URL = *urlBase + r.DateTime + "/index.html"
return r
}
// End should be called when the tests are complete
func (r *Report) End() {
r.Duration = time.Since(r.StartTime)
sort.Sort(r.Failed)
sort.Sort(r.Passed)
r.Runs = []ReportRun{
{Name: "Failed", Runs: r.Failed},
{Name: "Passed", Runs: r.Passed},
}
}
// AllPassed returns true if there were no failed tests
func (r *Report) AllPassed() bool {
return len(r.Failed) == 0
}
// RecordResult should be called with a Run when it has finished to be
// recorded into the Report
func (r *Report) RecordResult(t *Run) {
if !t.passed() {
r.Failed = append(r.Failed, t)
} else {
r.Passed = append(r.Passed, t)
}
}
// Title returns a human readable summary title for the Report
func (r *Report) Title() string {
if r.AllPassed() {
return fmt.Sprintf("PASS: All tests finished OK in %v", r.Duration)
}
return fmt.Sprintf("FAIL: %d tests failed in %v", len(r.Failed), r.Duration)
}
// LogSummary writes the summary to the log file
func (r *Report) LogSummary() {
log.Printf("Logs in %q", r.LogDir)
// Summarise results
log.Printf("SUMMARY")
log.Println(r.Title())
if !r.AllPassed() {
for _, t := range r.Failed {
log.Printf(" * %s", toShell(t.nextCmdLine()))
log.Printf(" * Failed tests: %v", t.failedTests)
}
}
}
// LogHTML writes the summary to index.html in LogDir
func (r *Report) LogHTML() {
r.IndexHTML = path.Join(r.LogDir, "index.html")
out, err := os.Create(r.IndexHTML)
if err != nil {
log.Fatalf("Failed to open index.html: %v", err)
}
defer func() {
err := out.Close()
if err != nil {
log.Fatalf("Failed to close index.html: %v", err)
}
}()
err = reportTemplate.Execute(out, r)
if err != nil {
log.Fatalf("Failed to execute template: %v", err)
}
_ = open.Start("file://" + r.IndexHTML)
}
var reportHTML = `<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="utf-8">
<title>{{ .Title }}</title>
<style>
table {
border-collapse: collapse;
border-spacing: 0;
border: 1px solid #ddd;
}
table.tests {
width: 100%;
}
table, th, td {
border: 1px solid black;
}
.Failed {
color: red;
}
.Passed {
color: green;
}
.false {
font-weight: lighter;
}
.true {
font-weight: bold;
}
th, td {
text-align: left;
padding: 4px;
}
tr:nth-child(even) {
background-color: #f2f2f2
}
</style>
</head>
<body>
<h1>{{ .Title }}</h1>
<table>
<tr><th>Version</th><td>{{ .Version }}</td></tr>
<tr><th>Date</th><td>{{ .DateTime}} [<a href="{{ .URL }}">online</a>]</td></tr>
<tr><th>Duration</th><td>{{ .Duration }}</td></tr>
{{ if .Previous}}<tr><th>Previous</th><td><a href="../{{ .Previous }}/index.html">{{ .Previous }}</a></td></tr>{{ end }}
<tr><th>Up</th><td><a href="../">Older Tests</a></td></tr>
</table>
{{ range .Runs }}
{{ if .Runs }}
<h2 class="{{ .Name }}">{{ .Name }}: {{ len .Runs }}</h2>
<table class="{{ .Name }} tests">
<tr>
<th>Backend</th>
<th>Remote</th>
<th>Test</th>
<th>SubDir</th>
<th>FastList</th>
<th>Failed</th>
<th>Logs</th>
</tr>
{{ $prevBackend := "" }}
{{ $prevRemote := "" }}
{{ range .Runs}}
<tr>
<td>{{ if ne $prevBackend .Backend }}{{ .Backend }}{{ end }}{{ $prevBackend = .Backend }}</td>
<td>{{ if ne $prevRemote .Remote }}{{ .Remote }}{{ end }}{{ $prevRemote = .Remote }}</td>
<td>{{ .Path }}</td>
<td><span class="{{ .SubDir }}">{{ .SubDir }}</span></td>
<td><span class="{{ .FastList }}">{{ .FastList }}</span></td>
<td>{{ .FailedTests }}</td>
<td>{{ range $i, $v := .Logs }}<a href="{{ $v }}">#{{ $i }}</a> {{ end }}</td>
</tr>
{{ end }}
</table>
{{ end }}
{{ end }}
</body>
</html>
`
var reportTemplate = template.Must(template.New("Report").Parse(reportHTML))
// EmailHTML sends the summary report to the email address supplied
func (r *Report) EmailHTML() {
if *emailReport == "" || r.IndexHTML == "" {
return
}
log.Printf("Sending email summary to %q", *emailReport)
cmdLine := []string{"mail", "-a", "Content-Type: text/html", *emailReport, "-s", "rclone integration tests: " + r.Title()}
cmd := exec.Command(cmdLine[0], cmdLine[1:]...)
in, err := os.Open(r.IndexHTML)
if err != nil {
log.Fatalf("Failed to open index.html: %v", err)
}
cmd.Stdin = in
cmd.Stdout = os.Stdout
cmd.Stderr = os.Stderr
err = cmd.Run()
if err != nil {
log.Fatalf("Failed to send email: %v", err)
}
_ = in.Close()
}
// Upload uploads a copy of the report online
func (r *Report) Upload() {
if *uploadPath == "" || r.IndexHTML == "" {
return
}
dst := path.Join(*uploadPath, r.DateTime)
log.Printf("Uploading results to %q", dst)
cmdLine := []string{"rclone", "copy", "-v", r.LogDir, dst}
cmd := exec.Command(cmdLine[0], cmdLine[1:]...)
cmd.Stdout = os.Stdout
cmd.Stderr = os.Stderr
err := cmd.Run()
if err != nil {
log.Fatalf("Failed to upload results: %v", err)
}
}

318
fstest/test_all/run.go Normal file
View file

@ -0,0 +1,318 @@
// Run a test
package main
import (
"bytes"
"fmt"
"go/build"
"io"
"log"
"os"
"os/exec"
"path"
"regexp"
"runtime"
"strings"
"time"
"github.com/ncw/rclone/fs"
)
const testBase = "github.com/ncw/rclone/"
// Run holds info about a running test
//
// A run just runs one command line, but it can be run multiple times
// if retries are needed.
type Run struct {
// Config
Remote string // name of the test remote
Backend string // name of the backend
Path string // path to the source directory
SubDir bool // add -sub-dir to tests
FastList bool // add -fast-list to tests
NoRetries bool // don't retry if set
// Internals
cmdLine []string
cmdString string
try int
err error
output []byte
failedTests []string
runFlag string
logDir string // directory to place the logs
trialName string // name/log file name of current trial
trialNames []string // list of all the trials
}
// Runs records multiple Run objects
type Runs []*Run
// Sort interface
func (rs Runs) Len() int { return len(rs) }
func (rs Runs) Swap(i, j int) { rs[i], rs[j] = rs[j], rs[i] }
func (rs Runs) Less(i, j int) bool {
a, b := rs[i], rs[j]
if a.Backend < b.Backend {
return true
} else if a.Backend > b.Backend {
return false
}
if a.Remote < b.Remote {
return true
} else if a.Remote > b.Remote {
return false
}
if a.Path < b.Path {
return true
} else if a.Path > b.Path {
return false
}
if !a.SubDir && b.SubDir {
return true
} else if a.SubDir && !b.SubDir {
return false
}
if !a.FastList && b.FastList {
return true
} else if a.FastList && !b.FastList {
return false
}
return false
}
// dumpOutput prints the error output
func (r *Run) dumpOutput() {
log.Println("------------------------------------------------------------")
log.Printf("---- %q ----", r.cmdString)
log.Println(string(r.output))
log.Println("------------------------------------------------------------")
}
var failRe = regexp.MustCompile(`(?m)^--- FAIL: (Test\w*) \(`)
// findFailures looks for all the tests which failed
func (r *Run) findFailures() {
oldFailedTests := r.failedTests
r.failedTests = nil
for _, matches := range failRe.FindAllSubmatch(r.output, -1) {
r.failedTests = append(r.failedTests, string(matches[1]))
}
if len(r.failedTests) != 0 {
r.runFlag = "^(" + strings.Join(r.failedTests, "|") + ")$"
} else {
r.runFlag = ""
}
if r.passed() && len(r.failedTests) != 0 {
log.Printf("%q - Expecting no errors but got: %v", r.cmdString, r.failedTests)
r.dumpOutput()
} else if !r.passed() && len(r.failedTests) == 0 {
log.Printf("%q - Expecting errors but got none: %v", r.cmdString, r.failedTests)
r.dumpOutput()
r.failedTests = oldFailedTests
}
}
// nextCmdLine returns the next command line
func (r *Run) nextCmdLine() []string {
cmdLine := r.cmdLine
if r.runFlag != "" {
cmdLine = append(cmdLine, "-test.run", r.runFlag)
}
return cmdLine
}
// trial runs a single test
func (r *Run) trial() {
cmdLine := r.nextCmdLine()
cmdString := toShell(cmdLine)
msg := fmt.Sprintf("%q - Starting (try %d/%d)", cmdString, r.try, *maxTries)
log.Println(msg)
logName := path.Join(r.logDir, r.trialName)
out, err := os.Create(logName)
if err != nil {
log.Fatalf("Couldn't create log file: %v", err)
}
defer func() {
err := out.Close()
if err != nil {
log.Fatalf("Failed to close log file: %v", err)
}
}()
_, _ = fmt.Fprintln(out, msg)
// Early exit if --try-run
if *dryRun {
log.Printf("Not executing as --dry-run: %v", cmdLine)
_, _ = fmt.Fprintln(out, "--dry-run is set - not running")
return
}
// Internal buffer
var b bytes.Buffer
multiOut := io.MultiWriter(out, &b)
cmd := exec.Command(cmdLine[0], cmdLine[1:]...)
cmd.Stderr = multiOut
cmd.Stdout = multiOut
start := time.Now()
r.err = cmd.Run()
r.output = b.Bytes()
duration := time.Since(start)
r.findFailures()
if r.passed() {
msg = fmt.Sprintf("%q - Finished OK in %v (try %d/%d)", cmdString, duration, r.try, *maxTries)
} else {
msg = fmt.Sprintf("%q - Finished ERROR in %v (try %d/%d): %v: Failed %v", cmdString, duration, r.try, *maxTries, r.err, r.failedTests)
}
log.Println(msg)
_, _ = fmt.Fprintln(out, msg)
}
// passed returns true if the test passed
func (r *Run) passed() bool {
return r.err == nil
}
// GOPATH returns the current GOPATH
func GOPATH() string {
gopath := os.Getenv("GOPATH")
if gopath == "" {
gopath = build.Default.GOPATH
}
return gopath
}
// BinaryName turns a package name into a binary name
func (r *Run) BinaryName() string {
binary := path.Base(r.Path) + ".test"
if runtime.GOOS == "windows" {
binary += ".exe"
}
return binary
}
// BinaryPath turns a package name into a binary path
func (r *Run) BinaryPath() string {
return path.Join(r.Path, r.BinaryName())
}
// PackagePath returns the path to the package
func (r *Run) PackagePath() string {
return path.Join(GOPATH(), "src", r.Path)
}
// Chdir into the package directory
func (r *Run) Chdir() {
err := os.Chdir(r.PackagePath())
if err != nil {
log.Fatalf("Failed to chdir to package %q: %v", r.Path, err)
}
}
// MakeTestBinary makes the binary we will run
func (r *Run) MakeTestBinary() {
binary := r.BinaryPath()
binaryName := r.BinaryName()
log.Printf("%s: Making test binary %q", r.Path, binaryName)
cmdLine := []string{"go", "test", "-c", "-o", binary, testBase + r.Path}
if *dryRun {
log.Printf("Not executing: %v", cmdLine)
return
}
err := exec.Command(cmdLine[0], cmdLine[1:]...).Run()
if err != nil {
log.Fatalf("Failed to make test binary: %v", err)
}
if _, err := os.Stat(binary); err != nil {
log.Fatalf("Couldn't find test binary %q", binary)
}
}
// RemoveTestBinary removes the binary made in makeTestBinary
func (r *Run) RemoveTestBinary() {
if *dryRun {
return
}
binary := r.BinaryPath()
err := os.Remove(binary) // Delete the binary when finished
if err != nil {
log.Printf("Error removing test binary %q: %v", binary, err)
}
}
// Name returns the run name as a file name friendly string
func (r *Run) Name() string {
ns := []string{
r.Backend,
strings.Replace(r.Path, "/", ".", -1),
r.Remote,
}
if r.SubDir {
ns = append(ns, "subdir")
}
if r.FastList {
ns = append(ns, "fastlist")
}
ns = append(ns, fmt.Sprintf("%d", r.try))
s := strings.Join(ns, "-")
s = strings.Replace(s, ":", "", -1)
return s
}
// Init the Run
func (r *Run) Init() {
binary := r.BinaryPath()
r.cmdLine = []string{binary, "-test.v", "-test.timeout", timeout.String(), "-remote", r.Remote}
r.try = 1
if *verbose {
r.cmdLine = append(r.cmdLine, "-verbose")
fs.Config.LogLevel = fs.LogLevelDebug
}
if *runOnly != "" {
r.cmdLine = append(r.cmdLine, "-test.run", *runOnly)
}
if r.SubDir {
r.cmdLine = append(r.cmdLine, "-subdir")
}
if r.FastList {
r.cmdLine = append(r.cmdLine, "-fast-list")
}
r.cmdString = toShell(r.cmdLine)
}
// Logs returns all the log names
func (r *Run) Logs() []string {
return r.trialNames
}
// FailedTests returns the failed tests as a comma separated string, limiting the number
func (r *Run) FailedTests() string {
const maxTests = 5
ts := r.failedTests
if len(ts) > maxTests {
ts = ts[:maxTests:maxTests]
ts = append(ts, fmt.Sprintf("… (%d more)", len(r.failedTests)-maxTests))
}
return strings.Join(ts, ", ")
}
// Run runs all the trials for this test
func (r *Run) Run(logDir string, result chan<- *Run) {
r.Init()
r.logDir = logDir
for r.try = 1; r.try <= *maxTries; r.try++ {
r.trialName = r.Name() + ".txt"
r.trialNames = append(r.trialNames, r.trialName)
log.Printf("Starting run with log %q", r.trialName)
r.trial()
if r.passed() || r.NoRetries {
break
}
}
if !r.passed() {
r.dumpOutput()
}
result <- r
}

View file

@ -4,24 +4,22 @@
// See the `test` target in the Makefile. // See the `test` target in the Makefile.
package main package main
/* FIXME
Make TesTrun have a []string of flags to try - that then makes it generic
*/
import ( import (
"flag" "flag"
"go/build"
"log" "log"
"os" "os"
"os/exec"
"path" "path"
"regexp" "regexp"
"runtime"
"strings" "strings"
"time" "time"
_ "github.com/ncw/rclone/backend/all" // import all fs _ "github.com/ncw/rclone/backend/all" // import all fs
"github.com/ncw/rclone/fs"
"github.com/ncw/rclone/fs/config"
"github.com/ncw/rclone/fs/list"
"github.com/ncw/rclone/fs/operations"
"github.com/ncw/rclone/fstest"
) )
type remoteConfig struct { type remoteConfig struct {
@ -31,218 +29,23 @@ type remoteConfig struct {
} }
var ( var (
remotes = []remoteConfig{
// {
// Name: "TestAmazonCloudDrive:",
// SubDir: false,
// FastList: false,
// },
{
Name: "TestB2:",
SubDir: true,
FastList: true,
},
{
Name: "TestCryptDrive:",
SubDir: false,
FastList: true,
},
{
Name: "TestCryptSwift:",
SubDir: false,
FastList: false,
},
{
Name: "TestDrive:",
SubDir: false,
FastList: true,
},
{
Name: "TestDropbox:",
SubDir: false,
FastList: false,
},
{
Name: "TestGoogleCloudStorage:",
SubDir: true,
FastList: true,
},
{
Name: "TestHubic:",
SubDir: false,
FastList: false,
},
{
Name: "TestJottacloud:",
SubDir: false,
FastList: true,
},
{
Name: "TestOneDrive:",
SubDir: false,
FastList: false,
},
{
Name: "TestS3:",
SubDir: true,
FastList: true,
},
{
Name: "TestSftp:",
SubDir: false,
FastList: false,
},
{
Name: "TestSwift:",
SubDir: true,
FastList: true,
},
{
Name: "TestYandex:",
SubDir: false,
FastList: false,
},
{
Name: "TestFTP:",
SubDir: false,
FastList: false,
},
{
Name: "TestBox:",
SubDir: false,
FastList: false,
},
{
Name: "TestQingStor:",
SubDir: false,
FastList: false,
},
{
Name: "TestAzureBlob:",
SubDir: true,
FastList: true,
},
{
Name: "TestPcloud:",
SubDir: false,
FastList: false,
},
{
Name: "TestWebdav:",
SubDir: false,
FastList: false,
},
{
Name: "TestCache:",
SubDir: false,
FastList: false,
},
{
Name: "TestMega:",
SubDir: false,
FastList: false,
},
{
Name: "TestOpenDrive:",
SubDir: false,
FastList: false,
},
{
Name: "TestUnion:",
SubDir: false,
FastList: false,
},
}
// Flags // Flags
maxTries = flag.Int("maxtries", 5, "Number of times to try each test") maxTries = flag.Int("maxtries", 5, "Number of times to try each test")
runTests = flag.String("remotes", "", "Comma separated list of remotes to test, eg 'TestSwift:,TestS3'") testRemotes = flag.String("remotes", "", "Comma separated list of remotes to test, eg 'TestSwift:,TestS3'")
clean = flag.Bool("clean", false, "Instead of testing, clean all left over test directories") testBackends = flag.String("backends", "", "Comma separated list of backends to test, eg 's3,googlecloudstorage")
runOnly = flag.String("run", "", "Run only those tests matching the regexp supplied") testTests = flag.String("tests", "", "Comma separated list of tests to test, eg 'fs/sync,fs/operations'")
timeout = flag.Duration("timeout", 30*time.Minute, "Maximum time to run each test for before giving up") clean = flag.Bool("clean", false, "Instead of testing, clean all left over test directories")
runOnly = flag.String("run", "", "Run only those tests matching the regexp supplied")
timeout = flag.Duration("timeout", 30*time.Minute, "Maximum time to run each test for before giving up")
configFile = flag.String("config", "fstest/test_all/config.yaml", "Path to config file")
outputDir = flag.String("output", path.Join(os.TempDir(), "rclone-integration-tests"), "Place to store results")
emailReport = flag.String("email", "", "Set to email the report to the address supplied")
dryRun = flag.Bool("dry-run", false, "Print commands which would be executed only")
urlBase = flag.String("url-base", "https://pub.rclone.org/integration-tests/", "Base for the online version")
uploadPath = flag.String("upload", "", "Set this to an rclone path to upload the results here")
verbose = flag.Bool("verbose", false, "Set to enable verbose logging in the tests")
) )
// test holds info about a running test
type test struct {
pkg string
remote string
subdir bool
cmdLine []string
cmdString string
try int
err error
output []byte
failedTests []string
runFlag string
}
// newTest creates a new test
func newTest(pkg, remote string, subdir bool, fastlist bool) *test {
binary := pkgBinary(pkg)
t := &test{
pkg: pkg,
remote: remote,
subdir: subdir,
cmdLine: []string{binary, "-test.timeout", timeout.String(), "-remote", remote},
try: 1,
}
if *fstest.Verbose {
t.cmdLine = append(t.cmdLine, "-test.v")
fs.Config.LogLevel = fs.LogLevelDebug
}
if *runOnly != "" {
t.cmdLine = append(t.cmdLine, "-test.run", *runOnly)
}
if subdir {
t.cmdLine = append(t.cmdLine, "-subdir")
}
if fastlist {
t.cmdLine = append(t.cmdLine, "-fast-list")
}
t.cmdString = toShell(t.cmdLine)
return t
}
// dumpOutput prints the error output
func (t *test) dumpOutput() {
log.Println("------------------------------------------------------------")
log.Printf("---- %q ----", t.cmdString)
log.Println(string(t.output))
log.Println("------------------------------------------------------------")
}
var failRe = regexp.MustCompile(`(?m)^--- FAIL: (Test\w*) \(`)
// findFailures looks for all the tests which failed
func (t *test) findFailures() {
oldFailedTests := t.failedTests
t.failedTests = nil
for _, matches := range failRe.FindAllSubmatch(t.output, -1) {
t.failedTests = append(t.failedTests, string(matches[1]))
}
if len(t.failedTests) != 0 {
t.runFlag = "^(" + strings.Join(t.failedTests, "|") + ")$"
} else {
t.runFlag = ""
}
if t.passed() && len(t.failedTests) != 0 {
log.Printf("%q - Expecting no errors but got: %v", t.cmdString, t.failedTests)
t.dumpOutput()
} else if !t.passed() && len(t.failedTests) == 0 {
log.Printf("%q - Expecting errors but got none: %v", t.cmdString, t.failedTests)
t.dumpOutput()
t.failedTests = oldFailedTests
}
}
// nextCmdLine returns the next command line
func (t *test) nextCmdLine() []string {
cmdLine := t.cmdLine
if t.runFlag != "" {
cmdLine = append(cmdLine, "-test.run", t.runFlag)
}
return cmdLine
}
// if matches then is definitely OK in the shell // if matches then is definitely OK in the shell
var shellOK = regexp.MustCompile("^[A-Za-z0-9./_:-]+$") var shellOK = regexp.MustCompile("^[A-Za-z0-9./_:-]+$")
@ -261,181 +64,53 @@ func toShell(args []string) (result string) {
return result return result
} }
// trial runs a single test
func (t *test) trial() {
cmdLine := t.nextCmdLine()
cmdString := toShell(cmdLine)
log.Printf("%q - Starting (try %d/%d)", cmdString, t.try, *maxTries)
cmd := exec.Command(cmdLine[0], cmdLine[1:]...)
start := time.Now()
t.output, t.err = cmd.CombinedOutput()
duration := time.Since(start)
t.findFailures()
if t.passed() {
log.Printf("%q - Finished OK in %v (try %d/%d)", cmdString, duration, t.try, *maxTries)
} else {
log.Printf("%q - Finished ERROR in %v (try %d/%d): %v: Failed %v", cmdString, duration, t.try, *maxTries, t.err, t.failedTests)
}
}
// cleanFs runs a single clean fs for left over directories
func (t *test) cleanFs() error {
f, err := fs.NewFs(t.remote)
if err != nil {
return err
}
entries, err := list.DirSorted(f, true, "")
if err != nil {
return err
}
return entries.ForDirError(func(dir fs.Directory) error {
remote := dir.Remote()
if fstest.MatchTestRemote.MatchString(remote) {
log.Printf("Purging %s%s", t.remote, remote)
dir, err := fs.NewFs(t.remote + remote)
if err != nil {
return err
}
return operations.Purge(dir, "")
}
return nil
})
}
// clean runs a single clean on a fs for left over directories
func (t *test) clean() {
log.Printf("%q - Starting clean (try %d/%d)", t.remote, t.try, *maxTries)
start := time.Now()
t.err = t.cleanFs()
if t.err != nil {
log.Printf("%q - Failed to purge %v", t.remote, t.err)
}
duration := time.Since(start)
if t.passed() {
log.Printf("%q - Finished OK in %v (try %d/%d)", t.cmdString, duration, t.try, *maxTries)
} else {
log.Printf("%q - Finished ERROR in %v (try %d/%d): %v", t.cmdString, duration, t.try, *maxTries, t.err)
}
}
// passed returns true if the test passed
func (t *test) passed() bool {
return t.err == nil
}
// run runs all the trials for this test
func (t *test) run(result chan<- *test) {
for t.try = 1; t.try <= *maxTries; t.try++ {
if *clean {
if !t.subdir {
t.clean()
}
} else {
t.trial()
}
if t.passed() {
break
}
}
if !t.passed() {
t.dumpOutput()
}
result <- t
}
// GOPATH returns the current GOPATH
func GOPATH() string {
gopath := os.Getenv("GOPATH")
if gopath == "" {
gopath = build.Default.GOPATH
}
return gopath
}
// turn a package name into a binary name
func pkgBinaryName(pkg string) string {
binary := path.Base(pkg) + ".test"
if runtime.GOOS == "windows" {
binary += ".exe"
}
return binary
}
// turn a package name into a binary path
func pkgBinary(pkg string) string {
return path.Join(pkgPath(pkg), pkgBinaryName(pkg))
}
// returns the path to the package
func pkgPath(pkg string) string {
return path.Join(GOPATH(), "src", pkg)
}
// cd into the package directory
func pkgChdir(pkg string) {
err := os.Chdir(pkgPath(pkg))
if err != nil {
log.Fatalf("Failed to chdir to package %q: %v", pkg, err)
}
}
// makeTestBinary makes the binary we will run
func makeTestBinary(pkg string) {
binaryName := pkgBinaryName(pkg)
log.Printf("%s: Making test binary %q", pkg, binaryName)
pkgChdir(pkg)
err := exec.Command("go", "test", "-c", "-o", binaryName).Run()
if err != nil {
log.Fatalf("Failed to make test binary: %v", err)
}
binary := pkgBinary(pkg)
if _, err := os.Stat(binary); err != nil {
log.Fatalf("Couldn't find test binary %q", binary)
}
}
// removeTestBinary removes the binary made in makeTestBinary
func removeTestBinary(pkg string) {
binary := pkgBinary(pkg)
err := os.Remove(binary) // Delete the binary when finished
if err != nil {
log.Printf("Error removing test binary %q: %v", binary, err)
}
}
func main() { func main() {
flag.Parse() flag.Parse()
packages := flag.Args() conf, err := NewConfig(*configFile)
log.Printf("Testing packages: %s", strings.Join(packages, ", ")) if err != nil {
if *runTests != "" { log.Println("test_all should be run from the root of the rclone source code")
newRemotes := []remoteConfig{} log.Fatal(err)
for _, name := range strings.Split(*runTests, ",") {
for i := range remotes {
if remotes[i].Name == name {
newRemotes = append(newRemotes, remotes[i])
goto found
}
}
log.Printf("Remote %q not found - inserting with default flags", name)
newRemotes = append(newRemotes, remoteConfig{Name: name})
found:
}
remotes = newRemotes
} }
// Filter selection
if *testRemotes != "" {
conf.filterBackendsByRemotes(strings.Split(*testRemotes, ","))
}
if *testBackends != "" {
conf.filterBackendsByBackends(strings.Split(*testBackends, ","))
}
if *testTests != "" {
conf.filterTests(strings.Split(*testTests, ","))
}
// Just clean the directories if required
if *clean {
err := cleanRemotes(conf.Remotes())
if err != nil {
log.Fatalf("Failed to clean: %v", err)
}
return
}
var names []string var names []string
for _, remote := range remotes { for _, remote := range conf.Backends {
names = append(names, remote.Name) names = append(names, remote.Remote)
} }
log.Printf("Testing remotes: %s", strings.Join(names, ", ")) log.Printf("Testing remotes: %s", strings.Join(names, ", "))
start := time.Now() // Runs we will do for this test
if *clean { runs := conf.MakeRuns()
config.LoadConfig()
packages = []string{"clean"} // Create Report
} else { report := NewReport()
for _, pkg := range packages {
makeTestBinary(pkg) // Make the test binaries, one per Path found in the tests
defer removeTestBinary(pkg) done := map[string]struct{}{}
for _, run := range runs {
if _, found := done[run.Path]; !found {
done[run.Path] = struct{}{}
run.MakeTestBinary()
defer run.RemoveTestBinary()
} }
} }
@ -443,46 +118,26 @@ func main() {
_ = os.Setenv("RCLONE_CACHE_DB_WAIT_TIME", "30m") _ = os.Setenv("RCLONE_CACHE_DB_WAIT_TIME", "30m")
// start the tests // start the tests
results := make(chan *test, 8) results := make(chan *Run, 8)
awaiting := 0 awaiting := 0
bools := []bool{false, true} for _, run := range runs {
if *clean { go run.Run(report.LogDir, results)
// Don't run -subdir and -fast-list if -clean awaiting++
bools = bools[:1]
}
for _, pkg := range packages {
for _, remote := range remotes {
for _, subdir := range bools {
for _, fastlist := range bools {
if (!subdir || subdir && remote.SubDir) && (!fastlist || fastlist && remote.FastList) {
go newTest(pkg, remote.Name, subdir, fastlist).run(results)
awaiting++
}
}
}
}
} }
// Wait for the tests to finish // Wait for the tests to finish
var failed []*test
for ; awaiting > 0; awaiting-- { for ; awaiting > 0; awaiting-- {
t := <-results t := <-results
if !t.passed() { report.RecordResult(t)
failed = append(failed, t)
}
} }
duration := time.Since(start)
// Summarise results // Log and exit
log.Printf("SUMMARY") report.End()
if len(failed) == 0 { report.LogSummary()
log.Printf("PASS: All tests finished OK in %v", duration) report.LogHTML()
} else { report.EmailHTML()
log.Printf("FAIL: %d tests failed in %v", len(failed), duration) report.Upload()
for _, t := range failed { if !report.AllPassed() {
log.Printf(" * %s", toShell(t.nextCmdLine()))
log.Printf(" * Failed tests: %v", t.failedTests)
}
os.Exit(1) os.Exit(1)
} }
} }