Merge pull request #887 from restic/rest-integration-tests

WIP: rest: Make backend honor the REST protocol

run_integration_tests.go

@@ -27,6 +27,7 @@ var ForbiddenImports = map[string]bool{
 
 var runCrossCompile = flag.Bool("cross-compile", true, "run cross compilation tests")
 var minioServer = flag.String("minio", "", "path to the minio server binary")
+var restServer = flag.String("rest", "", "path to the rest-server binary")
 var debug = flag.Bool("debug", false, "output debug messages")
 
 var minioServerEnv = map[string]string{
@@ -35,9 +36,10 @@ var minioServerEnv = map[string]string{
 }
 
 var minioEnv = map[string]string{
-	"RESTIC_TEST_S3_SERVER": "http://127.0.0.1:9000",
+	"RESTIC_TEST_S3_SERVER":   "http://127.0.0.1:9000",
-	"AWS_ACCESS_KEY_ID":     "KEBIYDZ87HCIH5D17YCN",
+	"RESTIC_TEST_REST_SERVER": "http://127.0.0.1:8000",
-	"AWS_SECRET_ACCESS_KEY": "bVX1KhipSBPopEfmhc7rGz8ooxx27xdJ7Gkh1mVe",
+	"AWS_ACCESS_KEY_ID":       "KEBIYDZ87HCIH5D17YCN",
+	"AWS_SECRET_ACCESS_KEY":   "bVX1KhipSBPopEfmhc7rGz8ooxx27xdJ7Gkh1mVe",
 }
 
 func init() {
@@ -54,11 +56,15 @@ type CIEnvironment interface {
 // TravisEnvironment is the environment in which Travis tests run.
 type TravisEnvironment struct {
 	goxOSArch []string
-	minio     string
 
+	minio        string
 	minioSrv     *Background
 	minioTempdir string
 
+	rest        string
+	restSrv     *Background
+	restTempdir string
+
 	env map[string]string
 }
 
@@ -118,7 +124,7 @@ func (env *TravisEnvironment) runMinio() error {
 
 	dir, err := ioutil.TempDir("", "minio-root")
 	if err != nil {
-		return fmt.Errorf("running minio server failed: %v", err)
+		return fmt.Errorf("TempDir: %v", err)
 	}
 
 	env.minioSrv, err = StartBackgroundCommand(minioServerEnv, env.minio,
@@ -142,6 +148,29 @@ func (env *TravisEnvironment) runMinio() error {
 	return nil
 }
 
+func (env *TravisEnvironment) runRESTServer() error {
+	if env.rest == "" {
+		return nil
+	}
+
+	// start rest server
+	msg("starting rest server at %s", env.rest)
+
+	dir, err := ioutil.TempDir("", "rest-server-root")
+	if err != nil {
+		return fmt.Errorf("TempDir: %v", err)
+	}
+
+	env.restSrv, err = StartBackgroundCommand(map[string]string{}, env.rest,
+		"--path", dir)
+	if err != nil {
+		return fmt.Errorf("error running rest server: %v", err)
+	}
+
+	env.restTempdir = dir
+	return nil
+}
+
 // Prepare installs dependencies and starts services in order to run the tests.
 func (env *TravisEnvironment) Prepare() error {
 	env.env = make(map[string]string)
@@ -151,6 +180,7 @@ func (env *TravisEnvironment) Prepare() error {
 	for _, pkg := range []string{
 		"golang.org/x/tools/cmd/cover",
 		"github.com/pierrre/gotestcover",
+		"github.com/restic/rest-server",
 	} {
 		err := run("go", "get", pkg)
 		if err != nil {
@@ -158,12 +188,17 @@ func (env *TravisEnvironment) Prepare() error {
 		}
 	}
 
+	env.rest = filepath.Join(os.Getenv("GOPATH"), "bin", "rest-server")
+
 	if err := env.getMinio(); err != nil {
 		return err
 	}
 	if err := env.runMinio(); err != nil {
 		return err
 	}
+	if err := env.runRESTServer(); err != nil {
+		return err
+	}
 
 	if *runCrossCompile && !(runtime.Version() < "go1.7") {
 		// only test cross compilation on linux with Travis
@@ -227,6 +262,29 @@ func (env *TravisEnvironment) Teardown() error {
 		}
 	}
 
+	if env.restSrv != nil {
+		msg("stopping rest-server\n")
+
+		if env.restSrv.Cmd.ProcessState == nil {
+			err := env.restSrv.Cmd.Process.Kill()
+			if err != nil {
+				fmt.Fprintf(os.Stderr, "error killing rest-server process: %v", err)
+			}
+		} else {
+			result := <-env.restSrv.Result
+			if result.Error != nil {
+				msg("rest-server returned error: %v\n", result.Error)
+				msg("stdout: %s\n", result.Stdout)
+				msg("stderr: %s\n", result.Stderr)
+			}
+		}
+
+		err := os.RemoveAll(env.restTempdir)
+		if err != nil {
+			msg("error removing rest-server tempdir %v: %v\n", env.restTempdir, err)
+		}
+	}
+
 	return nil
 }
 

src/cmds/restic/global.go

@@ -379,7 +379,7 @@ func create(s string) (restic.Backend, error) {
 		debug.Log("create s3 repository at %#v", loc.Config)
 		return s3.Open(cfg)
 	case "rest":
-		return rest.Open(loc.Config.(rest.Config))
+		return rest.Create(loc.Config.(rest.Config))
 	}
 
 	debug.Log("invalid repository scheme: %v", s)

src/restic/backend/rest/rest.go

@@ -69,6 +69,45 @@ func Open(cfg Config) (restic.Backend, error) {
 	return &restBackend{url: cfg.URL, connChan: connChan, client: client}, nil
 }
 
+// Create creates a new REST on server configured in config.
+func Create(cfg Config) (restic.Backend, error) {
+	be, err := Open(cfg)
+	if err != nil {
+		return nil, err
+	}
+
+	_, err = be.Stat(restic.Handle{Type: restic.ConfigFile})
+	if err == nil {
+		return nil, errors.Fatal("config file already exists")
+	}
+
+	url := *cfg.URL
+	values := url.Query()
+	values.Set("create", "true")
+	url.RawQuery = values.Encode()
+
+	resp, err := http.Post(url.String(), "binary/octet-stream", strings.NewReader(""))
+	if err != nil {
+		return nil, err
+	}
+
+	if resp.StatusCode != http.StatusOK {
+		return nil, errors.Fatalf("server response unexpected: %v (%v)", resp.Status, resp.StatusCode)
+	}
+
+	_, err = io.Copy(ioutil.Discard, resp.Body)
+	if err != nil {
+		return nil, err
+	}
+
+	err = resp.Body.Close()
+	if err != nil {
+		return nil, err
+	}
+
+	return be, nil
+}
+
 // Location returns this backend's location (the server's URL).
 func (b *restBackend) Location() string {
 	return b.url.String()
@@ -103,6 +142,7 @@ func (b *restBackend) Save(h restic.Handle, rd io.Reader) (err error) {
 		return errors.Wrap(err, "client.Post")
 	}
 
+	// fmt.Printf("status is %v (%v)\n", resp.Status, resp.StatusCode)
 	if resp.StatusCode != 200 {
 		return errors.Errorf("unexpected HTTP response code %v", resp.StatusCode)
 	}
@@ -222,7 +262,7 @@ func (b *restBackend) Remove(h restic.Handle) error {
 	}
 
 	if resp.StatusCode != 200 {
-		return errors.New("blob not removed")
+		return errors.Errorf("blob not removed, server response: %v (%v)", resp.Status, resp.StatusCode)
 	}
 
 	io.Copy(ioutil.Discard, resp.Body)

src/restic/backend/rest/rest_test.go

@@ -6,8 +6,6 @@ import (
 	"os"
 	"restic"
 
-	"restic/errors"
-
 	"restic/backend/rest"
 	"restic/backend/test"
 	. "restic/test"
@@ -32,21 +30,7 @@ func init() {
 	}
 
 	test.CreateFn = func() (restic.Backend, error) {
-		be, err := rest.Open(cfg)
+		return rest.Create(cfg)
-		if err != nil {
-			return nil, err
-		}
-
-		exists, err := be.Test(restic.Handle{Type: restic.ConfigFile, Name: ""})
-		if err != nil {
-			return nil, err
-		}
-
-		if exists {
-			return nil, errors.New("config already exists")
-		}
-
-		return be, nil
 	}
 
 	test.OpenFn = func() (restic.Backend, error) {

src/restic/backend/test/tests.go

@@ -375,7 +375,7 @@ var filenameTests = []struct {
 	data string
 }{
 	{"1dfc6bc0f06cb255889e9ea7860a5753e8eb9665c9a96627971171b444e3113e", "x"},
-	{"foobar", "foobar"},
+	{"f00b4r", "foobar"},
 	{
 		"1dfc6bc0f06cb255889e9ea7860a5753e8eb9665c9a96627971171b444e3113e4bf8f2d9144cc5420a80f04a4880ad6155fc58903a4fb6457c476c43541dcaa6-5",
 		"foobar content of data blob",
@@ -500,11 +500,11 @@ func TestBackend(t testing.TB) {
 		ts := testStrings[0]
 
 		// create blob
-		err := b.Save(restic.Handle{Type: tpe, Name: ts.id}, strings.NewReader(ts.data))
+		h := restic.Handle{Type: tpe, Name: ts.id}
-		test.Assert(t, err != nil, "expected error, got %v", err)
+		err := b.Save(h, strings.NewReader(ts.data))
+		test.Assert(t, err != nil, "expected error for %v, got %v", h, err)
 
 		// remove and recreate
-		h := restic.Handle{Type: tpe, Name: ts.id}
 		err = b.Remove(h)
 		test.OK(t, err)