diff --git a/.travis.yml b/.travis.yml index 304962c3a..ce8630344 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,8 +4,8 @@ sudo: false go: - 1.3.3 - 1.4.3 - - 1.5.3 - - 1.6 + - 1.5.4 + - 1.6.1 os: - linux @@ -18,7 +18,7 @@ matrix: - os: osx go: 1.4.3 - os: osx - go: 1.5.3 + go: 1.5.4 notifications: irc: diff --git a/Vagrantfile b/Vagrantfile index bac6e5a7c..19c73673b 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -35,7 +35,6 @@ end def packages_darwin return <<-EOF ruby -e "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install)" - brew install caskroom/cask/brew-cask brew cask install osxfuse EOF end diff --git a/appveyor.yml b/appveyor.yml index 10933757e..7b2720df0 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -13,8 +13,8 @@ init: install: - rmdir c:\go /s /q - - appveyor DownloadFile https://storage.googleapis.com/golang/go1.6.windows-amd64.msi - - msiexec /i go1.6.windows-amd64.msi /q + - appveyor DownloadFile https://storage.googleapis.com/golang/go1.6.1.windows-amd64.msi + - msiexec /i go1.6.1.windows-amd64.msi /q - go version - go env - appveyor DownloadFile http://sourceforge.netcologne.de/project/gnuwin32/tar/1.13-1/tar-1.13-1-bin.zip -FileName tar.zip diff --git a/run_integration_tests.go b/run_integration_tests.go index 519a6cd1f..2f2c71284 100644 --- a/run_integration_tests.go +++ b/run_integration_tests.go @@ -1,4 +1,4 @@ -// +build ignore +// xbuild ignore package main @@ -15,38 +15,56 @@ import ( "regexp" "runtime" "strings" - "sync" ) var runCrossCompile = flag.Bool("cross-compile", true, "run cross compilation tests") var minioServer = flag.String("minio", "", "path to the minio server binary") +var debug = flag.Bool("debug", false, "output debug messages") + +var minioServerEnv = map[string]string{ + "MINIO_ACCESS_KEY": "KEBIYDZ87HCIH5D17YCN", + "MINIO_SECRET_KEY": "bVX1KhipSBPopEfmhc7rGz8ooxx27xdJ7Gkh1mVe", +} + +var minioEnv = map[string]string{ + "RESTIC_TEST_S3_SERVER": "http://127.0.0.1:9000", + "AWS_ACCESS_KEY_ID": "KEBIYDZ87HCIH5D17YCN", + "AWS_SECRET_ACCESS_KEY": "bVX1KhipSBPopEfmhc7rGz8ooxx27xdJ7Gkh1mVe", +} func init() { flag.Parse() } +// CIEnvironment is implemented by environments where tests can be run. type CIEnvironment interface { - Prepare() - RunTests() + Prepare() error + RunTests() error + Teardown() error } +// TravisEnvironment is the environment in which Travis tests run. type TravisEnvironment struct { goxArch []string goxOS []string minio string + + minioSrv *Background + minioTempdir string + + env map[string]string } -func (env *TravisEnvironment) getMinio() { +func (env *TravisEnvironment) getMinio() error { if *minioServer != "" { msg("using minio server at %q\n", *minioServer) env.minio = *minioServer - return + return nil } tempfile, err := ioutil.TempFile("", "minio-server-") if err != nil { - fmt.Fprintf(os.Stderr, "create tempfile failed: %v\n", err) - os.Exit(10) + return fmt.Errorf("create tempfile for minio download failed: %v\n", err) } url := fmt.Sprintf("https://dl.minio.io/server/minio/release/%s-%s/minio", @@ -54,56 +72,108 @@ func (env *TravisEnvironment) getMinio() { msg("downloading %v\n", url) res, err := http.Get(url) if err != nil { - msg("downloading minio failed: %v\n", err) - return + return fmt.Errorf("error downloading minio server: %v\n", err) } _, err = io.Copy(tempfile, res.Body) if err != nil { - msg("downloading minio failed: %v\n", err) - return + return fmt.Errorf("error saving minio server to file: %v\n", err) } err = res.Body.Close() if err != nil { - msg("saving minio failed: %v\n", err) - return + return fmt.Errorf("error closing HTTP download: %v\n", err) } err = tempfile.Close() if err != nil { msg("closing tempfile failed: %v\n", err) - return + return fmt.Errorf("error closing minio server file: %v\n", err) } err = os.Chmod(tempfile.Name(), 0755) if err != nil { - msg("making minio server executable failed: %v\n", err) - return + return fmt.Errorf("chmod(minio-server) failed: %v", err) } msg("downloaded minio server to %v\n", tempfile.Name()) env.minio = tempfile.Name() + return nil } -func (env *TravisEnvironment) Prepare() { +func (env *TravisEnvironment) runMinio() error { + if env.minio == "" { + return nil + } + + // start minio server + msg("starting minio server at %s", env.minio) + + dir, err := ioutil.TempDir("", "minio-root") + if err != nil { + return fmt.Errorf("running minio server failed: %v", err) + } + + env.minioSrv, err = StartBackgroundCommand(minioServerEnv, env.minio, + "server", + "--address", "127.0.0.1:9000", + dir) + if err != nil { + return fmt.Errorf("error running minio server: %v", err) + } + + // go func() { + // time.Sleep(300 * time.Millisecond) + // env.minioSrv.Cmd.Process.Kill() + // }() + + for k, v := range minioEnv { + env.env[k] = v + } + + env.minioTempdir = dir + return nil +} + +// Prepare installs dependencies and starts services in order to run the tests. +func (env *TravisEnvironment) Prepare() error { + env.env = make(map[string]string) + msg("preparing environment for Travis CI\n") - run("go", "get", "golang.org/x/tools/cmd/cover") - run("go", "get", "github.com/mattn/goveralls") - run("go", "get", "github.com/pierrre/gotestcover") - env.getMinio() + for _, pkg := range []string{ + "golang.org/x/tools/cmd/cover", + "github.com/mattn/goveralls", + "github.com/pierrre/gotestcover", + } { + err := run("go", "get", pkg) + if err != nil { + return err + } + } + + if err := env.getMinio(); err != nil { + return err + } + if err := env.runMinio(); err != nil { + return err + } if runtime.GOOS == "darwin" { // install the libraries necessary for fuse - run("brew", "update") - run("brew", "install", "caskroom/cask/brew-cask") - run("brew", "cask", "install", "osxfuse") + if err := run("brew", "update"); err != nil { + return err + } + if err := run("brew", "cask", "install", "osxfuse"); err != nil { + return err + } } if *runCrossCompile { // only test cross compilation on linux with Travis - run("go", "get", "github.com/mitchellh/gox") + if err := run("go", "get", "github.com/mitchellh/gox"); err != nil { + return err + } if runtime.GOOS == "linux" { env.goxArch = []string{"386", "amd64"} if !strings.HasPrefix(runtime.Version(), "go1.3") { @@ -120,11 +190,46 @@ func (env *TravisEnvironment) Prepare() { v := runtime.Version() if !strings.HasPrefix(v, "go1.5") && !strings.HasPrefix(v, "go1.6") { - run("gox", "-build-toolchain", + err := run("gox", "-build-toolchain", "-os", strings.Join(env.goxOS, " "), "-arch", strings.Join(env.goxArch, " ")) + + if err != nil { + return err + } } } + + return nil +} + +// Teardown stops backend services and cleans the environment again. +func (env *TravisEnvironment) Teardown() error { + msg("run travis teardown\n") + if env.minioSrv != nil { + msg("stopping minio server\n") + + if env.minioSrv.Cmd.ProcessState == nil { + err := env.minioSrv.Cmd.Process.Kill() + if err != nil { + fmt.Fprintf(os.Stderr, "error killing minio server process: %v", err) + } + } else { + result := <-env.minioSrv.Result + if result.Error != nil { + msg("minio server returned error: %v\n", result.Error) + msg("stdout: %s\n", result.Stdout) + msg("stderr: %s\n", result.Stderr) + } + } + + err := os.RemoveAll(env.minioTempdir) + if err != nil { + msg("error removing minio tempdir %v: %v\n", env.minioTempdir, err) + } + } + + return nil } func goVersionAtLeast151() bool { @@ -141,13 +246,63 @@ func goVersionAtLeast151() bool { return true } -type MinioServer struct { - cmd *exec.Cmd - done bool - m sync.Mutex +// Background is a program running in the background. +type Background struct { + Cmd *exec.Cmd + Result chan Result } -func (env *TravisEnvironment) RunTests() { +// Result is the result of a program that ran in the background. +type Result struct { + Stdout, Stderr string + Error error +} + +// StartBackgroundCommand runs a program in the background. +func StartBackgroundCommand(env map[string]string, cmd string, args ...string) (*Background, error) { + msg("running background command %v %v\n", cmd, args) + b := Background{ + Result: make(chan Result, 1), + } + + stdout := bytes.NewBuffer(nil) + stderr := bytes.NewBuffer(nil) + + c := exec.Command(cmd, args...) + c.Stdout = stdout + c.Stderr = stderr + + if *debug { + c.Stdout = io.MultiWriter(c.Stdout, os.Stdout) + c.Stderr = io.MultiWriter(c.Stderr, os.Stderr) + } + c.Env = updateEnv(os.Environ(), env) + + b.Cmd = c + + err := c.Start() + if err != nil { + msg("error starting background job %v: %v\n", cmd, err) + return nil, err + } + + go func() { + err := b.Cmd.Wait() + msg("background job %v returned: %v\n", cmd, err) + msg("stdout: %s\n", stdout.Bytes()) + msg("stderr: %s\n", stderr.Bytes()) + b.Result <- Result{ + Stdout: string(stdout.Bytes()), + Stderr: string(stderr.Bytes()), + Error: err, + } + }() + + return &b, nil +} + +// RunTests starts the tests for Travis. +func (env *TravisEnvironment) RunTests() error { // run fuse tests on darwin if runtime.GOOS != "darwin" { msg("skip fuse integration tests on %v\n", runtime.GOOS) @@ -156,17 +311,15 @@ func (env *TravisEnvironment) RunTests() { cwd, err := os.Getwd() if err != nil { - fmt.Fprintf(os.Stderr, "Getwd() returned error: %v", err) - os.Exit(9) + return fmt.Errorf("Getwd() returned error: %v", err) } - envWithGOPATH := make(map[string]string) - envWithGOPATH["GOPATH"] = cwd + ":" + filepath.Join(cwd, "vendor") + env.env["GOPATH"] = cwd + ":" + filepath.Join(cwd, "vendor") if *runCrossCompile { // compile for all target architectures with tags for _, tags := range []string{"release", "debug"} { - runWithEnv(envWithGOPATH, "gox", "-verbose", + runWithEnv(env.env, "gox", "-verbose", "-os", strings.Join(env.goxOS, " "), "-arch", strings.Join(env.goxArch, " "), "-tags", tags, @@ -176,37 +329,36 @@ func (env *TravisEnvironment) RunTests() { } // run the build script - run("go", "run", "build.go") - - var srv *MinioServer - if env.minio != "" { - srv, err = NewMinioServer(env.minio) - if err != nil { - fmt.Fprintf(os.Stderr, "error running minio server: %v", err) - os.Exit(8) - } - - for k, v := range minioEnv { - envWithGOPATH[k] = v - } + if err := run("go", "run", "build.go"); err != nil { + return err } // run the tests and gather coverage information - runWithEnv(envWithGOPATH, "gotestcover", "-coverprofile", "all.cov", "cmds/...", "restic/...") + err = runWithEnv(env.env, "gotestcover", "-coverprofile", "all.cov", "cmds/...", "restic/...") + if err != nil { + return err + } - runGofmt() - - srv.Stop() + return runGofmt() } +// AppveyorEnvironment is the environment on Windows. type AppveyorEnvironment struct{} -func (env *AppveyorEnvironment) Prepare() { +// Prepare installs dependencies and starts services in order to run the tests. +func (env *AppveyorEnvironment) Prepare() error { msg("preparing environment for Appveyor CI\n") + return nil } -func (env *AppveyorEnvironment) RunTests() { - run("go", "run", "build.go", "-v", "-T") +// RunTests start the tests. +func (env *AppveyorEnvironment) RunTests() error { + return run("go", "run", "build.go", "-v", "-T") +} + +// Teardown is a noop. +func (env *AppveyorEnvironment) Teardown() error { + return nil } // findGoFiles returns a list of go source code file names below dir. @@ -255,17 +407,15 @@ func updateEnv(env []string, override map[string]string) []string { return newEnv } -func runGofmt() { +func runGofmt() error { dir, err := os.Getwd() if err != nil { - fmt.Fprintf(os.Stderr, "Getwd(): %v\n", err) - os.Exit(5) + return fmt.Errorf("Getwd(): %v\n", err) } files, err := findGoFiles(dir) if err != nil { - fmt.Fprintf(os.Stderr, "error finding Go files: %v\n", err) - os.Exit(4) + return fmt.Errorf("error finding Go files: %v\n", err) } msg("runGofmt() with %d files\n", len(files)) @@ -275,26 +425,24 @@ func runGofmt() { buf, err := cmd.Output() if err != nil { - fmt.Fprintf(os.Stderr, "error running gofmt: %v", err) - fmt.Fprintf(os.Stderr, "output:\n%s\n", buf) - os.Exit(3) + return fmt.Errorf("error running gofmt: %v\noutput: %s\n", err, buf) } if len(buf) > 0 { - fmt.Fprintf(os.Stderr, "not formatted with `gofmt`:\n") - fmt.Fprintln(os.Stderr, string(buf)) - os.Exit(6) + return fmt.Errorf("not formatted with `gofmt`:\n%s\n", buf) } + + return nil } -func run(command string, args ...string) { +func run(command string, args ...string) error { msg("run %v %v\n", command, strings.Join(args, " ")) - runWithEnv(nil, command, args...) + return runWithEnv(nil, command, args...) } // runWithEnv calls a command with the current environment, except the entries // of the env map are set additionally. -func runWithEnv(env map[string]string, command string, args ...string) { +func runWithEnv(env map[string]string, command string, args ...string) error { msg("runWithEnv %v %v\n", command, strings.Join(args, " ")) cmd := exec.Command(command, args...) cmd.Stdout = os.Stdout @@ -305,104 +453,10 @@ func runWithEnv(env map[string]string, command string, args ...string) { err := cmd.Run() if err != nil { - fmt.Fprintf(os.Stderr, "error running %v %v: %v", + return fmt.Errorf("error running %v %v: %v", command, strings.Join(args, " "), err) - os.Exit(3) - } -} - -var minioConfig = ` -{ - "version": "2", - "credentials": { - "accessKeyId": "KEBIYDZ87HCIH5D17YCN", - "secretAccessKey": "bVX1KhipSBPopEfmhc7rGz8ooxx27xdJ7Gkh1mVe" - } -} -` - -var minioEnv = map[string]string{ - "RESTIC_TEST_S3_SERVER": "http://127.0.0.1:9000", - "AWS_ACCESS_KEY_ID": "KEBIYDZ87HCIH5D17YCN", - "AWS_SECRET_ACCESS_KEY": "bVX1KhipSBPopEfmhc7rGz8ooxx27xdJ7Gkh1mVe", -} - -// NewMinioServer prepares and runs a minio server for the s3 backend tests in -// a temporary directory. -func NewMinioServer(minio string) (*MinioServer, error) { - msg("running minio server\n") - cfgdir, err := ioutil.TempDir("", "minio-config-") - if err != nil { - return nil, err - } - - cfg, err := os.Create(filepath.Join(cfgdir, "config.json")) - if err != nil { - return nil, err - } - - _, err = cfg.Write([]byte(minioConfig)) - if err != nil { - return nil, err - } - - err = cfg.Close() - if err != nil { - return nil, err - } - - dir, err := ioutil.TempDir("", "minio-root") - if err != nil { - return nil, err - } - - out := bytes.NewBuffer(nil) - - cmd := exec.Command(minio, - "--config-folder", cfgdir, - "--address", "127.0.0.1:9000", - "server", dir) - cmd.Stdout = out - cmd.Stderr = out - err = cmd.Start() - if err != nil { - return nil, err - } - - srv := &MinioServer{cmd: cmd} - go srv.Wait() - - return srv, nil -} - -func (m *MinioServer) Stop() { - if m == nil { - return - } - - msg("stopping minio server\n") - m.m.Lock() - m.done = true - m.m.Unlock() - err := m.cmd.Process.Kill() - if err != nil { - fmt.Fprintf(os.Stderr, "error stopping minio server: %v", err) - os.Exit(8) - } -} - -func (m *MinioServer) Wait() { - err := m.cmd.Wait() - msg("minio server exited\n") - m.m.Lock() - done := m.done - m.m.Unlock() - - if err != nil && !done { - fmt.Fprintf(os.Stderr, "error running minio server: %#v, output:\n", err) - // io.Copy(os.Stderr, out) - os.Exit(12) } + return nil } func isTravis() bool { @@ -426,7 +480,16 @@ func main() { os.Exit(1) } - for _, f := range []func(){env.Prepare, env.RunTests} { - f() + foundError := false + for _, f := range []func() error{env.Prepare, env.RunTests, env.Teardown} { + err := f() + if err != nil { + foundError = true + fmt.Fprintf(os.Stderr, "error: %v\n", err) + } + } + + if foundError { + os.Exit(1) } } diff --git a/src/restic/backend/test/tests.go b/src/restic/backend/test/tests.go index 60c35ffb0..ef0efd77a 100644 --- a/src/restic/backend/test/tests.go +++ b/src/restic/backend/test/tests.go @@ -77,7 +77,6 @@ func TestCreate(t testing.TB) { be, err := CreateFn() if err != nil { - fmt.Printf("foo\n") t.Fatalf("Create returned error: %v", err) }