TrueCloudLab/restic
16
0
Fork 2
Code Issues Pull requests 1 Projects Releases Packages Wiki Activity Actions

Merge pull request #498 from restic/fix-travis-drawin

Browse source 
Fix CI tests
This commit is contained in:
Alexander Neumann 2016-04-18 20:58:45 +02:00
commit 1eb896ae6f
5 changed files with 237 additions and 176 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
.travis.yml
View file

@ -4,8 +4,8 @@ sudo: false
go:
- 1.3.3
- 1.4.3
- 1.5.3
- 1.6
- 1.5.4
- 1.6.1
os:
- linux
@ -18,7 +18,7 @@ matrix:
- os: osx
go: 1.4.3
- os: osx
go: 1.5.3
go: 1.5.4
notifications:
irc:

1
Vagrantfile vendored
View file

@ -35,7 +35,6 @@ end
def packages_darwin
return <<-EOF
ruby -e "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install)"
brew install caskroom/cask/brew-cask
brew cask install osxfuse
EOF
end

4
appveyor.yml
View file

@ -13,8 +13,8 @@ init:
install:
- rmdir c:\go /s /q
- appveyor DownloadFile https://storage.googleapis.com/golang/go1.6.windows-amd64.msi
- msiexec /i go1.6.windows-amd64.msi /q
- appveyor DownloadFile https://storage.googleapis.com/golang/go1.6.1.windows-amd64.msi
- msiexec /i go1.6.1.windows-amd64.msi /q
- go version
- go env
- appveyor DownloadFile http://sourceforge.netcologne.de/project/gnuwin32/tar/1.13-1/tar-1.13-1-bin.zip -FileName tar.zip

401
run_integration_tests.go
View file

@ -1,4 +1,4 @@
// +build ignore
// xbuild ignore
package main
@ -15,38 +15,56 @@ import (
"regexp"
"runtime"
"strings"
"sync"
)
var runCrossCompile = flag.Bool("cross-compile", true, "run cross compilation tests")
var minioServer = flag.String("minio", "", "path to the minio server binary")
var debug = flag.Bool("debug", false, "output debug messages")
var minioServerEnv = map[string]string{
"MINIO_ACCESS_KEY": "KEBIYDZ87HCIH5D17YCN",
"MINIO_SECRET_KEY": "bVX1KhipSBPopEfmhc7rGz8ooxx27xdJ7Gkh1mVe",
}
var minioEnv = map[string]string{
"RESTIC_TEST_S3_SERVER": "http://127.0.0.1:9000",
"AWS_ACCESS_KEY_ID": "KEBIYDZ87HCIH5D17YCN",
"AWS_SECRET_ACCESS_KEY": "bVX1KhipSBPopEfmhc7rGz8ooxx27xdJ7Gkh1mVe",
}
func init() {
flag.Parse()
}
// CIEnvironment is implemented by environments where tests can be run.
type CIEnvironment interface {
Prepare()
RunTests()
Prepare() error
RunTests() error
Teardown() error
}
// TravisEnvironment is the environment in which Travis tests run.
type TravisEnvironment struct {
goxArch []string
goxOS []string
minio string
minioSrv *Background
minioTempdir string
env map[string]string
}
func (env *TravisEnvironment) getMinio() {
func (env *TravisEnvironment) getMinio() error {
if *minioServer != "" {
msg("using minio server at %q\n", *minioServer)
env.minio = *minioServer
return
return nil
}
tempfile, err := ioutil.TempFile("", "minio-server-")
if err != nil {
fmt.Fprintf(os.Stderr, "create tempfile failed: %v\n", err)
os.Exit(10)
return fmt.Errorf("create tempfile for minio download failed: %v\n", err)
}
url := fmt.Sprintf("https://dl.minio.io/server/minio/release/%s-%s/minio",
@ -54,56 +72,108 @@ func (env *TravisEnvironment) getMinio() {
msg("downloading %v\n", url)
res, err := http.Get(url)
if err != nil {
msg("downloading minio failed: %v\n", err)
return
return fmt.Errorf("error downloading minio server: %v\n", err)
}
_, err = io.Copy(tempfile, res.Body)
if err != nil {
msg("downloading minio failed: %v\n", err)
return
return fmt.Errorf("error saving minio server to file: %v\n", err)
}
err = res.Body.Close()
if err != nil {
msg("saving minio failed: %v\n", err)
return
return fmt.Errorf("error closing HTTP download: %v\n", err)
}
err = tempfile.Close()
if err != nil {
msg("closing tempfile failed: %v\n", err)
return
return fmt.Errorf("error closing minio server file: %v\n", err)
}
err = os.Chmod(tempfile.Name(), 0755)
if err != nil {
msg("making minio server executable failed: %v\n", err)
return
return fmt.Errorf("chmod(minio-server) failed: %v", err)
}
msg("downloaded minio server to %v\n", tempfile.Name())
env.minio = tempfile.Name()
return nil
}
func (env *TravisEnvironment) Prepare() {
func (env *TravisEnvironment) runMinio() error {
if env.minio == "" {
return nil
}
// start minio server
msg("starting minio server at %s", env.minio)
dir, err := ioutil.TempDir("", "minio-root")
if err != nil {
return fmt.Errorf("running minio server failed: %v", err)
}
env.minioSrv, err = StartBackgroundCommand(minioServerEnv, env.minio,
"server",
"--address", "127.0.0.1:9000",
dir)
if err != nil {
return fmt.Errorf("error running minio server: %v", err)
}
// go func() {
// time.Sleep(300 * time.Millisecond)
// env.minioSrv.Cmd.Process.Kill()
// }()
for k, v := range minioEnv {
env.env[k] = v
}
env.minioTempdir = dir
return nil
}
// Prepare installs dependencies and starts services in order to run the tests.
func (env *TravisEnvironment) Prepare() error {
env.env = make(map[string]string)
msg("preparing environment for Travis CI\n")
run("go", "get", "golang.org/x/tools/cmd/cover")
run("go", "get", "github.com/mattn/goveralls")
run("go", "get", "github.com/pierrre/gotestcover")
env.getMinio()
for _, pkg := range []string{
"golang.org/x/tools/cmd/cover",
"github.com/mattn/goveralls",
"github.com/pierrre/gotestcover",
} {
err := run("go", "get", pkg)
if err != nil {
return err
}
}
if err := env.getMinio(); err != nil {
return err
}
if err := env.runMinio(); err != nil {
return err
}
if runtime.GOOS == "darwin" {
// install the libraries necessary for fuse
run("brew", "update")
run("brew", "install", "caskroom/cask/brew-cask")
run("brew", "cask", "install", "osxfuse")
if err := run("brew", "update"); err != nil {
return err
}
if err := run("brew", "cask", "install", "osxfuse"); err != nil {
return err
}
}
if *runCrossCompile {
// only test cross compilation on linux with Travis
run("go", "get", "github.com/mitchellh/gox")
if err := run("go", "get", "github.com/mitchellh/gox"); err != nil {
return err
}
if runtime.GOOS == "linux" {
env.goxArch = []string{"386", "amd64"}
if !strings.HasPrefix(runtime.Version(), "go1.3") {
@ -120,11 +190,46 @@ func (env *TravisEnvironment) Prepare() {
v := runtime.Version()
if !strings.HasPrefix(v, "go1.5") && !strings.HasPrefix(v, "go1.6") {
run("gox", "-build-toolchain",
err := run("gox", "-build-toolchain",
"-os", strings.Join(env.goxOS, " "),
"-arch", strings.Join(env.goxArch, " "))
if err != nil {
return err
}
}
}
return nil
}
// Teardown stops backend services and cleans the environment again.
func (env *TravisEnvironment) Teardown() error {
msg("run travis teardown\n")
if env.minioSrv != nil {
msg("stopping minio server\n")
if env.minioSrv.Cmd.ProcessState == nil {
err := env.minioSrv.Cmd.Process.Kill()
if err != nil {
fmt.Fprintf(os.Stderr, "error killing minio server process: %v", err)
}
} else {
result := <-env.minioSrv.Result
if result.Error != nil {
msg("minio server returned error: %v\n", result.Error)
msg("stdout: %s\n", result.Stdout)
msg("stderr: %s\n", result.Stderr)
}
}
err := os.RemoveAll(env.minioTempdir)
if err != nil {
msg("error removing minio tempdir %v: %v\n", env.minioTempdir, err)
}
}
return nil
}
func goVersionAtLeast151() bool {
@ -141,13 +246,63 @@ func goVersionAtLeast151() bool {
return true
}
type MinioServer struct {
cmd *exec.Cmd
done bool
m sync.Mutex
// Background is a program running in the background.
type Background struct {
Cmd *exec.Cmd
Result chan Result
}
func (env *TravisEnvironment) RunTests() {
// Result is the result of a program that ran in the background.
type Result struct {
Stdout, Stderr string
Error error
}
// StartBackgroundCommand runs a program in the background.
func StartBackgroundCommand(env map[string]string, cmd string, args ...string) (*Background, error) {
msg("running background command %v %v\n", cmd, args)
b := Background{
Result: make(chan Result, 1),
}
stdout := bytes.NewBuffer(nil)
stderr := bytes.NewBuffer(nil)
c := exec.Command(cmd, args...)
c.Stdout = stdout
c.Stderr = stderr
if *debug {
c.Stdout = io.MultiWriter(c.Stdout, os.Stdout)
c.Stderr = io.MultiWriter(c.Stderr, os.Stderr)
}
c.Env = updateEnv(os.Environ(), env)
b.Cmd = c
err := c.Start()
if err != nil {
msg("error starting background job %v: %v\n", cmd, err)
return nil, err
}
go func() {
err := b.Cmd.Wait()
msg("background job %v returned: %v\n", cmd, err)
msg("stdout: %s\n", stdout.Bytes())
msg("stderr: %s\n", stderr.Bytes())
b.Result <- Result{
Stdout: string(stdout.Bytes()),
Stderr: string(stderr.Bytes()),
Error: err,
}
}()
return &b, nil
}
// RunTests starts the tests for Travis.
func (env *TravisEnvironment) RunTests() error {
// run fuse tests on darwin
if runtime.GOOS != "darwin" {
msg("skip fuse integration tests on %v\n", runtime.GOOS)
@ -156,17 +311,15 @@ func (env *TravisEnvironment) RunTests() {
cwd, err := os.Getwd()
if err != nil {
fmt.Fprintf(os.Stderr, "Getwd() returned error: %v", err)
os.Exit(9)
return fmt.Errorf("Getwd() returned error: %v", err)
}
envWithGOPATH := make(map[string]string)
envWithGOPATH["GOPATH"] = cwd + ":" + filepath.Join(cwd, "vendor")
env.env["GOPATH"] = cwd + ":" + filepath.Join(cwd, "vendor")
if *runCrossCompile {
// compile for all target architectures with tags
for _, tags := range []string{"release", "debug"} {
runWithEnv(envWithGOPATH, "gox", "-verbose",
runWithEnv(env.env, "gox", "-verbose",
"-os", strings.Join(env.goxOS, " "),
"-arch", strings.Join(env.goxArch, " "),
"-tags", tags,
@ -176,37 +329,36 @@ func (env *TravisEnvironment) RunTests() {
}
// run the build script
run("go", "run", "build.go")
var srv *MinioServer
if env.minio != "" {
srv, err = NewMinioServer(env.minio)
if err != nil {
fmt.Fprintf(os.Stderr, "error running minio server: %v", err)
os.Exit(8)
}
for k, v := range minioEnv {
envWithGOPATH[k] = v
}
if err := run("go", "run", "build.go"); err != nil {
return err
}
// run the tests and gather coverage information
runWithEnv(envWithGOPATH, "gotestcover", "-coverprofile", "all.cov", "cmds/...", "restic/...")
err = runWithEnv(env.env, "gotestcover", "-coverprofile", "all.cov", "cmds/...", "restic/...")
if err != nil {
return err
}
runGofmt()
srv.Stop()
return runGofmt()
}
// AppveyorEnvironment is the environment on Windows.
type AppveyorEnvironment struct{}
func (env *AppveyorEnvironment) Prepare() {
// Prepare installs dependencies and starts services in order to run the tests.
func (env *AppveyorEnvironment) Prepare() error {
msg("preparing environment for Appveyor CI\n")
return nil
}
func (env *AppveyorEnvironment) RunTests() {
run("go", "run", "build.go", "-v", "-T")
// RunTests start the tests.
func (env *AppveyorEnvironment) RunTests() error {
return run("go", "run", "build.go", "-v", "-T")
}
// Teardown is a noop.
func (env *AppveyorEnvironment) Teardown() error {
return nil
}
// findGoFiles returns a list of go source code file names below dir.
@ -255,17 +407,15 @@ func updateEnv(env []string, override map[string]string) []string {
return newEnv
}
func runGofmt() {
func runGofmt() error {
dir, err := os.Getwd()
if err != nil {
fmt.Fprintf(os.Stderr, "Getwd(): %v\n", err)
os.Exit(5)
return fmt.Errorf("Getwd(): %v\n", err)
}
files, err := findGoFiles(dir)
if err != nil {
fmt.Fprintf(os.Stderr, "error finding Go files: %v\n", err)
os.Exit(4)
return fmt.Errorf("error finding Go files: %v\n", err)
}
msg("runGofmt() with %d files\n", len(files))
@ -275,26 +425,24 @@ func runGofmt() {
buf, err := cmd.Output()
if err != nil {
fmt.Fprintf(os.Stderr, "error running gofmt: %v", err)
fmt.Fprintf(os.Stderr, "output:\n%s\n", buf)
os.Exit(3)
return fmt.Errorf("error running gofmt: %v\noutput: %s\n", err, buf)
}
if len(buf) > 0 {
fmt.Fprintf(os.Stderr, "not formatted with `gofmt`:\n")
fmt.Fprintln(os.Stderr, string(buf))
os.Exit(6)
return fmt.Errorf("not formatted with `gofmt`:\n%s\n", buf)
}
return nil
}
func run(command string, args ...string) {
func run(command string, args ...string) error {
msg("run %v %v\n", command, strings.Join(args, " "))
runWithEnv(nil, command, args...)
return runWithEnv(nil, command, args...)
}
// runWithEnv calls a command with the current environment, except the entries
// of the env map are set additionally.
func runWithEnv(env map[string]string, command string, args ...string) {
func runWithEnv(env map[string]string, command string, args ...string) error {
msg("runWithEnv %v %v\n", command, strings.Join(args, " "))
cmd := exec.Command(command, args...)
cmd.Stdout = os.Stdout
@ -305,104 +453,10 @@ func runWithEnv(env map[string]string, command string, args ...string) {
err := cmd.Run()
if err != nil {
fmt.Fprintf(os.Stderr, "error running %v %v: %v",
return fmt.Errorf("error running %v %v: %v",
command, strings.Join(args, " "), err)
os.Exit(3)
}
}
var minioConfig = `
{
"version": "2",
"credentials": {
"accessKeyId": "KEBIYDZ87HCIH5D17YCN",
"secretAccessKey": "bVX1KhipSBPopEfmhc7rGz8ooxx27xdJ7Gkh1mVe"
}
}
`
var minioEnv = map[string]string{
"RESTIC_TEST_S3_SERVER": "http://127.0.0.1:9000",
"AWS_ACCESS_KEY_ID": "KEBIYDZ87HCIH5D17YCN",
"AWS_SECRET_ACCESS_KEY": "bVX1KhipSBPopEfmhc7rGz8ooxx27xdJ7Gkh1mVe",
}
// NewMinioServer prepares and runs a minio server for the s3 backend tests in
// a temporary directory.
func NewMinioServer(minio string) (*MinioServer, error) {
msg("running minio server\n")
cfgdir, err := ioutil.TempDir("", "minio-config-")
if err != nil {
return nil, err
}
cfg, err := os.Create(filepath.Join(cfgdir, "config.json"))
if err != nil {
return nil, err
}
_, err = cfg.Write([]byte(minioConfig))
if err != nil {
return nil, err
}
err = cfg.Close()
if err != nil {
return nil, err
}
dir, err := ioutil.TempDir("", "minio-root")
if err != nil {
return nil, err
}
out := bytes.NewBuffer(nil)
cmd := exec.Command(minio,
"--config-folder", cfgdir,
"--address", "127.0.0.1:9000",
"server", dir)
cmd.Stdout = out
cmd.Stderr = out
err = cmd.Start()
if err != nil {
return nil, err
}
srv := &MinioServer{cmd: cmd}
go srv.Wait()
return srv, nil
}
func (m *MinioServer) Stop() {
if m == nil {
return
}
msg("stopping minio server\n")
m.m.Lock()
m.done = true
m.m.Unlock()
err := m.cmd.Process.Kill()
if err != nil {
fmt.Fprintf(os.Stderr, "error stopping minio server: %v", err)
os.Exit(8)
}
}
func (m *MinioServer) Wait() {
err := m.cmd.Wait()
msg("minio server exited\n")
m.m.Lock()
done := m.done
m.m.Unlock()
if err != nil && !done {
fmt.Fprintf(os.Stderr, "error running minio server: %#v, output:\n", err)
// io.Copy(os.Stderr, out)
os.Exit(12)
}
return nil
}
func isTravis() bool {
@ -426,7 +480,16 @@ func main() {
os.Exit(1)
}
for _, f := range []func(){env.Prepare, env.RunTests} {
f()
foundError := false
for _, f := range []func() error{env.Prepare, env.RunTests, env.Teardown} {
err := f()
if err != nil {
foundError = true
fmt.Fprintf(os.Stderr, "error: %v\n", err)
}
}
if foundError {
os.Exit(1)
}
}

1
src/restic/backend/test/tests.go
View file

@ -77,7 +77,6 @@ func TestCreate(t testing.TB) {
be, err := CreateFn()
if err != nil {
fmt.Printf("foo\n")
t.Fatalf("Create returned error: %v", err)
}