2018-01-12 16:30:54 +00:00
|
|
|
// Run tests for all the remotes. Run this with package names which
|
|
|
|
// need integration testing.
|
2015-12-30 00:18:31 +00:00
|
|
|
//
|
2018-01-12 16:30:54 +00:00
|
|
|
// See the `test` target in the Makefile.
|
2015-12-30 00:18:31 +00:00
|
|
|
package main
|
|
|
|
|
2018-09-29 13:48:29 +00:00
|
|
|
/* FIXME
|
|
|
|
|
|
|
|
Make TesTrun have a []string of flags to try - that then makes it generic
|
|
|
|
|
|
|
|
*/
|
|
|
|
|
2015-12-30 00:18:31 +00:00
|
|
|
import (
|
|
|
|
"flag"
|
|
|
|
"log"
|
|
|
|
"os"
|
2018-01-12 16:30:54 +00:00
|
|
|
"path"
|
2016-04-26 07:11:37 +00:00
|
|
|
"regexp"
|
2015-12-30 00:18:31 +00:00
|
|
|
"strings"
|
|
|
|
"time"
|
2016-01-03 21:49:26 +00:00
|
|
|
|
2018-01-12 16:30:54 +00:00
|
|
|
_ "github.com/ncw/rclone/backend/all" // import all fs
|
2015-12-30 00:18:31 +00:00
|
|
|
)
|
|
|
|
|
2017-06-06 22:04:01 +00:00
|
|
|
type remoteConfig struct {
|
|
|
|
Name string
|
|
|
|
SubDir bool
|
|
|
|
FastList bool
|
|
|
|
}
|
|
|
|
|
2015-12-30 00:18:31 +00:00
|
|
|
var (
|
|
|
|
// Flags
|
2018-09-29 13:48:29 +00:00
|
|
|
maxTries = flag.Int("maxtries", 5, "Number of times to try each test")
|
|
|
|
testRemotes = flag.String("remotes", "", "Comma separated list of remotes to test, eg 'TestSwift:,TestS3'")
|
|
|
|
testBackends = flag.String("backends", "", "Comma separated list of backends to test, eg 's3,googlecloudstorage")
|
|
|
|
testTests = flag.String("tests", "", "Comma separated list of tests to test, eg 'fs/sync,fs/operations'")
|
|
|
|
clean = flag.Bool("clean", false, "Instead of testing, clean all left over test directories")
|
|
|
|
runOnly = flag.String("run", "", "Run only those tests matching the regexp supplied")
|
|
|
|
timeout = flag.Duration("timeout", 30*time.Minute, "Maximum time to run each test for before giving up")
|
|
|
|
configFile = flag.String("config", "fstest/test_all/config.yaml", "Path to config file")
|
|
|
|
outputDir = flag.String("output", path.Join(os.TempDir(), "rclone-integration-tests"), "Place to store results")
|
|
|
|
emailReport = flag.String("email", "", "Set to email the report to the address supplied")
|
|
|
|
dryRun = flag.Bool("dry-run", false, "Print commands which would be executed only")
|
|
|
|
urlBase = flag.String("url-base", "https://pub.rclone.org/integration-tests/", "Base for the online version")
|
|
|
|
uploadPath = flag.String("upload", "", "Set this to an rclone path to upload the results here")
|
|
|
|
verbose = flag.Bool("verbose", false, "Set to enable verbose logging in the tests")
|
2015-12-30 00:18:31 +00:00
|
|
|
)
|
|
|
|
|
2017-06-13 12:19:18 +00:00
|
|
|
// if matches then is definitely OK in the shell
|
|
|
|
var shellOK = regexp.MustCompile("^[A-Za-z0-9./_:-]+$")
|
|
|
|
|
|
|
|
// converts a argv style input into a shell command
|
|
|
|
func toShell(args []string) (result string) {
|
|
|
|
for _, arg := range args {
|
|
|
|
if result != "" {
|
|
|
|
result += " "
|
|
|
|
}
|
|
|
|
if shellOK.MatchString(arg) {
|
|
|
|
result += arg
|
|
|
|
} else {
|
|
|
|
result += "'" + arg + "'"
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return result
|
|
|
|
}
|
|
|
|
|
2018-09-29 13:48:29 +00:00
|
|
|
func main() {
|
|
|
|
flag.Parse()
|
|
|
|
conf, err := NewConfig(*configFile)
|
2018-01-12 16:30:54 +00:00
|
|
|
if err != nil {
|
2018-09-29 13:48:29 +00:00
|
|
|
log.Println("test_all should be run from the root of the rclone source code")
|
|
|
|
log.Fatal(err)
|
2018-01-12 16:30:54 +00:00
|
|
|
}
|
|
|
|
|
2018-09-29 13:48:29 +00:00
|
|
|
// Filter selection
|
|
|
|
if *testRemotes != "" {
|
|
|
|
conf.filterBackendsByRemotes(strings.Split(*testRemotes, ","))
|
2015-12-30 00:18:31 +00:00
|
|
|
}
|
2018-09-29 13:48:29 +00:00
|
|
|
if *testBackends != "" {
|
|
|
|
conf.filterBackendsByBackends(strings.Split(*testBackends, ","))
|
2015-12-30 00:18:31 +00:00
|
|
|
}
|
2018-09-29 13:48:29 +00:00
|
|
|
if *testTests != "" {
|
|
|
|
conf.filterTests(strings.Split(*testTests, ","))
|
2015-12-30 00:18:31 +00:00
|
|
|
}
|
|
|
|
|
2018-09-29 13:48:29 +00:00
|
|
|
// Just clean the directories if required
|
|
|
|
if *clean {
|
|
|
|
err := cleanRemotes(conf.Remotes())
|
|
|
|
if err != nil {
|
|
|
|
log.Fatalf("Failed to clean: %v", err)
|
2017-06-06 22:04:01 +00:00
|
|
|
}
|
2018-09-29 13:48:29 +00:00
|
|
|
return
|
2015-12-30 00:18:31 +00:00
|
|
|
}
|
2018-09-29 13:48:29 +00:00
|
|
|
|
2017-06-06 22:04:01 +00:00
|
|
|
var names []string
|
2018-09-29 13:48:29 +00:00
|
|
|
for _, remote := range conf.Backends {
|
|
|
|
names = append(names, remote.Remote)
|
2017-06-06 22:04:01 +00:00
|
|
|
}
|
|
|
|
log.Printf("Testing remotes: %s", strings.Join(names, ", "))
|
2015-12-30 00:18:31 +00:00
|
|
|
|
2018-09-29 13:48:29 +00:00
|
|
|
// Runs we will do for this test
|
|
|
|
runs := conf.MakeRuns()
|
|
|
|
|
|
|
|
// Create Report
|
|
|
|
report := NewReport()
|
|
|
|
|
|
|
|
// Make the test binaries, one per Path found in the tests
|
|
|
|
done := map[string]struct{}{}
|
|
|
|
for _, run := range runs {
|
|
|
|
if _, found := done[run.Path]; !found {
|
|
|
|
done[run.Path] = struct{}{}
|
|
|
|
run.MakeTestBinary()
|
|
|
|
defer run.RemoveTestBinary()
|
2018-01-12 16:30:54 +00:00
|
|
|
}
|
2016-01-03 21:49:26 +00:00
|
|
|
}
|
2015-12-30 00:18:31 +00:00
|
|
|
|
2018-03-08 12:17:29 +00:00
|
|
|
// workaround for cache backend as we run simultaneous tests
|
|
|
|
_ = os.Setenv("RCLONE_CACHE_DB_WAIT_TIME", "30m")
|
|
|
|
|
2015-12-30 00:18:31 +00:00
|
|
|
// start the tests
|
2018-09-29 13:48:29 +00:00
|
|
|
results := make(chan *Run, 8)
|
2015-12-30 00:18:31 +00:00
|
|
|
awaiting := 0
|
2018-09-29 13:48:29 +00:00
|
|
|
for _, run := range runs {
|
|
|
|
go run.Run(report.LogDir, results)
|
|
|
|
awaiting++
|
2015-12-30 00:18:31 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
// Wait for the tests to finish
|
|
|
|
for ; awaiting > 0; awaiting-- {
|
|
|
|
t := <-results
|
2018-09-29 13:48:29 +00:00
|
|
|
report.RecordResult(t)
|
2015-12-30 00:18:31 +00:00
|
|
|
}
|
|
|
|
|
2018-09-29 13:48:29 +00:00
|
|
|
// Log and exit
|
|
|
|
report.End()
|
|
|
|
report.LogSummary()
|
|
|
|
report.LogHTML()
|
|
|
|
report.EmailHTML()
|
|
|
|
report.Upload()
|
|
|
|
if !report.AllPassed() {
|
2015-12-30 00:18:31 +00:00
|
|
|
os.Exit(1)
|
|
|
|
}
|
|
|
|
}
|