Implement package-local tests

backend/handle.go

@@ -0,0 +1,48 @@
+package backend
+
+import (
+	"errors"
+	"fmt"
+)
+
+// Handle is used to store and access data in a backend.
+type Handle struct {
+	Type Type
+	Name string
+}
+
+func (h Handle) String() string {
+	name := h.Name
+	if len(name) > 10 {
+		name = name[:10]
+	}
+	return fmt.Sprintf("<%s/%s>", h.Type, name)
+}
+
+// Valid returns an error if h is not valid.
+func (h Handle) Valid() error {
+	if h.Type == "" {
+		return errors.New("type is empty")
+	}
+
+	switch h.Type {
+	case Data:
+	case Key:
+	case Lock:
+	case Snapshot:
+	case Index:
+	case Config:
+	default:
+		return fmt.Errorf("invalid config %q", h.Type)
+	}
+
+	if h.Type == Config {
+		return nil
+	}
+
+	if h.Name == "" {
+		return errors.New("invalid Name")
+	}
+
+	return nil
+}

backend/handle_test.go

@@ -0,0 +1,28 @@
+package backend
+
+import "testing"
+
+var handleTests = []struct {
+	h     Handle
+	valid bool
+}{
+	{Handle{Name: "foo"}, false},
+	{Handle{Type: "foobar"}, false},
+	{Handle{Type: Config, Name: ""}, true},
+	{Handle{Type: Data, Name: ""}, false},
+	{Handle{Type: "", Name: "x"}, false},
+	{Handle{Type: Lock, Name: "010203040506"}, true},
+}
+
+func TestHandleValid(t *testing.T) {
+	for i, test := range handleTests {
+		err := test.h.Valid()
+		if err != nil && test.valid {
+			t.Errorf("test %v failed: error returned for valid handle: %v", i, err)
+		}
+
+		if !test.valid && err == nil {
+			t.Errorf("test %v failed: expected error for invalid handle not found", i)
+		}
+	}
+}

backend/interface.go

@@ -1,9 +1,6 @@
 package backend
 
-import (
-	"fmt"
-	"io"
-)
+import "io"
 
 // Type is the type of a Blob.
 type Type string
@@ -18,20 +15,6 @@ const (
 	Config        = "config"
 )
 
-// Handle is used to store and access data in a backend.
-type Handle struct {
-	Type Type
-	Name string
-}
-
-func (h Handle) String() string {
-	name := h.Name
-	if len(name) > 10 {
-		name = name[:10]
-	}
-	return fmt.Sprintf("<%s/%s>", h.Type, name)
-}
-
 // Backend is used to store and access data.
 type Backend interface {
 	// Location returns a string that describes the type and location of the

backend/local/backend_test.go

@@ -0,0 +1,20 @@
+// DO NOT EDIT!
+// generated at 2016-23-01 17:06:38 +0100 CET
+package local_test
+
+import (
+	"testing"
+
+	"github.com/restic/restic/backend/test"
+)
+
+func TestCreate(t *testing.T)    { test.Create(t) }
+func TestOpen(t *testing.T)      { test.Open(t) }
+func TestLocation(t *testing.T)  { test.Location(t) }
+func TestConfig(t *testing.T)    { test.Config(t) }
+func TestGetReader(t *testing.T) { test.GetReader(t) }
+func TestLoad(t *testing.T)      { test.Load(t) }
+func TestWrite(t *testing.T)     { test.Write(t) }
+func TestGeneric(t *testing.T)   { test.Generic(t) }
+func TestDelete(t *testing.T)    { test.Delete(t) }
+func TestCleanup(t *testing.T)   { test.Cleanup(t) }

backend/local/local.go

@@ -226,6 +226,10 @@ func (b *Local) GetReader(t backend.Type, name string, offset, length uint) (io.
 // Load returns the data stored in the backend for h at the given offset
 // and saves it in p. Load has the same semantics as io.ReaderAt.
 func (b *Local) Load(h backend.Handle, p []byte, off int64) (n int, err error) {
+	if err := h.Valid(); err != nil {
+		return 0, err
+	}
+
 	f, err := os.Open(filename(b.p, h.Type, h.Name))
 	if err != nil {
 		return 0, err

backend/local/local_test.go

@@ -0,0 +1,53 @@
+package local_test
+
+import (
+	"errors"
+	"fmt"
+	"io/ioutil"
+	"os"
+
+	"github.com/restic/restic/backend"
+	"github.com/restic/restic/backend/local"
+	"github.com/restic/restic/backend/test"
+)
+
+var tempBackendDir string
+
+//go:generate go run ../test/generate_backend_tests.go
+
+func init() {
+	test.CreateFn = func() (backend.Backend, error) {
+		if tempBackendDir != "" {
+			return nil, errors.New("temporary local backend dir already exists")
+		}
+
+		tempdir, err := ioutil.TempDir("", "restic-local-test-")
+		if err != nil {
+			return nil, err
+		}
+
+		fmt.Printf("created new test backend at %v\n", tempdir)
+		tempBackendDir = tempdir
+
+		return local.Create(tempdir)
+	}
+
+	test.OpenFn = func() (backend.Backend, error) {
+		if tempBackendDir == "" {
+			return nil, errors.New("repository not initialized")
+		}
+
+		return local.Open(tempBackendDir)
+	}
+
+	test.CleanupFn = func() error {
+		if tempBackendDir == "" {
+			return nil
+		}
+
+		fmt.Printf("removing test backend at %v\n", tempBackendDir)
+		err := os.RemoveAll(tempBackendDir)
+		tempBackendDir = ""
+		return err
+	}
+}

backend/test/generate_backend_tests.go

@@ -0,0 +1,111 @@
+// +build ignore
+
+package main
+
+import (
+	"bufio"
+	"flag"
+	"fmt"
+	"io"
+	"log"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"regexp"
+	"text/template"
+	"time"
+)
+
+var data struct {
+	Package   string
+	Funcs     []string
+	Timestamp string
+}
+
+var testTemplate = `
+// DO NOT EDIT!
+// generated at {{ .Timestamp }}
+package {{ .Package }}
+
+import (
+	"testing"
+
+	"github.com/restic/restic/backend/test"
+)
+
+{{ range $f := .Funcs }}func Test{{ $f }}(t *testing.T){ test.{{ $f }}(t) }
+{{ end }}
+`
+
+var testFile = flag.String("testfile", "../test/tests.go", "file to search test functions in")
+var outputFile = flag.String("output", "backend_test.go", "output file to write generated code to")
+
+func errx(err error) {
+	if err == nil {
+		return
+	}
+
+	fmt.Fprintf(os.Stderr, "error: %v\n", err)
+	os.Exit(1)
+}
+
+var funcRegex = regexp.MustCompile(`^func\s+([A-Z].*)\s*\(`)
+
+func findTestFunctions() (funcs []string) {
+	f, err := os.Open(*testFile)
+	errx(err)
+
+	sc := bufio.NewScanner(f)
+	for sc.Scan() {
+		match := funcRegex.FindStringSubmatch(sc.Text())
+		if len(match) > 0 {
+			funcs = append(funcs, match[1])
+		}
+	}
+
+	if err := sc.Err(); err != nil {
+		log.Fatalf("Error scanning file: %v", err)
+	}
+
+	errx(f.Close())
+	return funcs
+}
+
+func generateOutput(wr io.Writer, data interface{}) {
+	t := template.Must(template.New("backendtest").Parse(testTemplate))
+
+	cmd := exec.Command("gofmt")
+	cmd.Stdout = wr
+	in, err := cmd.StdinPipe()
+	errx(err)
+	errx(cmd.Start())
+	errx(t.Execute(in, data))
+	errx(in.Close())
+	errx(cmd.Wait())
+}
+
+func init() {
+	flag.Parse()
+}
+
+func main() {
+	dir, err := os.Getwd()
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "Getwd() %v\n", err)
+		os.Exit(1)
+	}
+
+	packageName := filepath.Base(dir)
+
+	f, err := os.Create(*outputFile)
+	errx(err)
+
+	data.Package = packageName + "_test"
+	data.Funcs = findTestFunctions()
+	data.Timestamp = time.Now().Format("2006-02-01 15:04:05 -0700 MST")
+	generateOutput(f, data)
+
+	errx(f.Close())
+
+	fmt.Printf("wrote backend tests for package %v\n", packageName)
+}

backend/test/tests.go

@@ -0,0 +1,550 @@
+package test
+
+import (
+	"bytes"
+	crand "crypto/rand"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"math/rand"
+	"reflect"
+	"sort"
+	"testing"
+
+	"github.com/restic/restic/backend"
+	. "github.com/restic/restic/test"
+)
+
+// CreateFn is a function that creates a temporary repository for the tests.
+var CreateFn func() (backend.Backend, error)
+
+// OpenFn is a function that opens a previously created temporary repository.
+var OpenFn func() (backend.Backend, error)
+
+// CleanupFn removes temporary files and directories created during the tests.
+var CleanupFn func() error
+
+var but backend.Backend // backendUnderTest
+var butInitialized bool
+
+func open(t testing.TB) backend.Backend {
+	if OpenFn == nil {
+		t.Fatal("OpenFn not set")
+	}
+
+	if CreateFn == nil {
+		t.Fatalf("CreateFn not set")
+	}
+
+	if !butInitialized {
+		be, err := CreateFn()
+		if err != nil {
+			t.Fatalf("Create returned unexpected error: %v", err)
+		}
+
+		but = be
+		butInitialized = true
+	}
+
+	if but == nil {
+		var err error
+		but, err = OpenFn()
+		if err != nil {
+			t.Fatalf("Open returned unexpected error: %v", err)
+		}
+	}
+
+	return but
+}
+
+func close(t testing.TB) {
+	if but == nil {
+		t.Fatalf("trying to close non-existing backend")
+	}
+
+	err := but.Close()
+	if err != nil {
+		t.Fatalf("Close returned unexpected error: %v", err)
+	}
+
+	but = nil
+}
+
+// Create creates a backend.
+func Create(t testing.TB) {
+	if CreateFn == nil {
+		t.Fatalf("CreateFn not set!")
+	}
+
+	be, err := CreateFn()
+	if err != nil {
+		fmt.Printf("foo\n")
+		t.Fatalf("Create returned error: %v", err)
+	}
+
+	butInitialized = true
+
+	err = be.Close()
+	if err != nil {
+		t.Fatalf("Close returned error: %v", err)
+	}
+}
+
+// Open opens a previously created backend.
+func Open(t testing.TB) {
+	if OpenFn == nil {
+		t.Fatalf("OpenFn not set!")
+	}
+
+	be, err := OpenFn()
+	if err != nil {
+		t.Fatalf("Open returned error: %v", err)
+	}
+
+	err = be.Close()
+	if err != nil {
+		t.Fatalf("Close returned error: %v", err)
+	}
+}
+
+// Location tests that a location string is returned.
+func Location(t testing.TB) {
+	b := open(t)
+	defer close(t)
+
+	l := b.Location()
+	if l == "" {
+		t.Fatalf("invalid location string %q", l)
+	}
+}
+
+// Config saves and loads a config from the backend.
+func Config(t testing.TB) {
+	b := open(t)
+	defer close(t)
+
+	var testString = "Config"
+
+	// create config and read it back
+	_, err := b.GetReader(backend.Config, "", 0, 0)
+	if err == nil {
+		t.Fatalf("did not get expected error for non-existing config")
+	}
+
+	blob, err := b.Create()
+	if err != nil {
+		t.Fatalf("Create() error: %v", err)
+	}
+
+	_, err = blob.Write([]byte(testString))
+	if err != nil {
+		t.Fatalf("Write() error: %v", err)
+	}
+
+	err = blob.Finalize(backend.Config, "")
+	if err != nil {
+		t.Fatalf("Finalize() error: %v", err)
+	}
+
+	// try accessing the config with different names, should all return the
+	// same config
+	for _, name := range []string{"", "foo", "bar", "0000000000000000000000000000000000000000000000000000000000000000"} {
+		rd, err := b.GetReader(backend.Config, name, 0, 0)
+		if err != nil {
+			t.Fatalf("unable to read config with name %q: %v", name, err)
+		}
+
+		buf, err := ioutil.ReadAll(rd)
+		if err != nil {
+			t.Fatalf("read config error: %v", err)
+		}
+
+		err = rd.Close()
+		if err != nil {
+			t.Fatalf("close error: %v", err)
+		}
+
+		if string(buf) != testString {
+			t.Fatalf("wrong data returned, want %q, got %q", testString, string(buf))
+		}
+	}
+}
+
+// GetReader tests various ways the GetReader function can be called.
+func GetReader(t testing.TB) {
+	b := open(t)
+	defer close(t)
+
+	length := rand.Intn(1<<24) + 2000
+
+	data := make([]byte, length)
+	_, err := io.ReadFull(crand.Reader, data)
+	OK(t, err)
+
+	blob, err := b.Create()
+	OK(t, err)
+
+	id := backend.Hash(data)
+
+	_, err = blob.Write([]byte(data))
+	OK(t, err)
+	OK(t, blob.Finalize(backend.Data, id.String()))
+
+	for i := 0; i < 500; i++ {
+		l := rand.Intn(length + 2000)
+		o := rand.Intn(length + 2000)
+
+		d := data
+		if o < len(d) {
+			d = d[o:]
+		} else {
+			o = len(d)
+			d = d[:0]
+		}
+
+		if l > 0 && l < len(d) {
+			d = d[:l]
+		}
+
+		rd, err := b.GetReader(backend.Data, id.String(), uint(o), uint(l))
+		OK(t, err)
+		buf, err := ioutil.ReadAll(rd)
+		OK(t, err)
+
+		if !bytes.Equal(buf, d) {
+			t.Fatalf("data not equal")
+		}
+	}
+
+	OK(t, b.Remove(backend.Data, id.String()))
+}
+
+// Load tests the backend's Load function.
+func Load(t testing.TB) {
+	b := open(t)
+	defer close(t)
+
+	_, err := b.Load(backend.Handle{}, nil, 0)
+	if err == nil {
+		t.Fatalf("Load() did not return an error for invalid handle")
+	}
+
+	_, err = b.Load(backend.Handle{Type: backend.Data, Name: "foobar"}, nil, 0)
+	if err == nil {
+		t.Fatalf("Load() did not return an error for non-existing blob")
+	}
+
+	length := rand.Intn(1<<24) + 2000
+
+	data := make([]byte, length)
+	_, err = io.ReadFull(crand.Reader, data)
+	if err != nil {
+		t.Fatalf("reading random data failed: %v", err)
+	}
+
+	id := backend.Hash(data)
+
+	blob, err := b.Create()
+	OK(t, err)
+
+	_, err = blob.Write([]byte(data))
+	OK(t, err)
+	OK(t, blob.Finalize(backend.Data, id.String()))
+
+	for i := 0; i < 500; i++ {
+		l := rand.Intn(length + 2000)
+		o := rand.Intn(length + 2000)
+
+		d := data
+		if o < len(d) {
+			d = d[o:]
+		} else {
+			o = len(d)
+			d = d[:0]
+		}
+
+		if l > 0 && l < len(d) {
+			d = d[:l]
+		}
+
+		buf := make([]byte, l)
+		h := backend.Handle{Type: backend.Data, Name: id.String()}
+		n, err := b.Load(h, buf, int64(o))
+
+		// if we requested data beyond the end of the file, ignore
+		// ErrUnexpectedEOF error
+		if l > len(d) && err == io.ErrUnexpectedEOF {
+			err = nil
+			buf = buf[:len(d)]
+		}
+
+		if err != nil {
+			t.Errorf("Load(%d, %d): unexpected error: %v", len(buf), int64(o), err)
+			continue
+		}
+
+		if n != len(buf) {
+			t.Errorf("Load(%d, %d): wrong length returned, want %d, got %d",
+				len(buf), int64(o), len(buf), n)
+			continue
+		}
+
+		buf = buf[:n]
+		if !bytes.Equal(buf, d) {
+			t.Errorf("Load(%d, %d) returned wrong bytes", len(buf), int64(o))
+			continue
+		}
+	}
+
+	OK(t, b.Remove(backend.Data, id.String()))
+}
+
+// Write tests writing data to the backend.
+func Write(t testing.TB) {
+	b := open(t)
+	defer close(t)
+
+	length := rand.Intn(1<<23) + 2000
+
+	data := make([]byte, length)
+	_, err := io.ReadFull(crand.Reader, data)
+	OK(t, err)
+	id := backend.Hash(data)
+
+	for i := 0; i < 10; i++ {
+		blob, err := b.Create()
+		OK(t, err)
+
+		o := 0
+		for o < len(data) {
+			l := rand.Intn(len(data) - o)
+			if len(data)-o < 20 {
+				l = len(data) - o
+			}
+
+			n, err := blob.Write(data[o : o+l])
+			OK(t, err)
+			if n != l {
+				t.Fatalf("wrong number of bytes written, want %v, got %v", l, n)
+			}
+
+			o += l
+		}
+
+		name := fmt.Sprintf("%s-%d", id, i)
+		OK(t, blob.Finalize(backend.Data, name))
+
+		rd, err := b.GetReader(backend.Data, name, 0, 0)
+		OK(t, err)
+
+		buf, err := ioutil.ReadAll(rd)
+		OK(t, err)
+
+		if len(buf) != len(data) {
+			t.Fatalf("number of bytes does not match, want %v, got %v", len(data), len(buf))
+		}
+
+		if !bytes.Equal(buf, data) {
+			t.Fatalf("data not equal")
+		}
+
+		err = b.Remove(backend.Data, name)
+		if err != nil {
+			t.Fatalf("error removing item: %v", err)
+		}
+	}
+}
+
+var testStrings = []struct {
+	id   string
+	data string
+}{
+	{"c3ab8ff13720e8ad9047dd39466b3c8974e592c2fa383d4a3960714caef0c4f2", "foobar"},
+	{"248d6a61d20638b8e5c026930c3e6039a33ce45964ff2167f6ecedd419db06c1", "abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq"},
+	{"cc5d46bdb4991c6eae3eb739c9c8a7a46fe9654fab79c47b4fe48383b5b25e1c", "foo/bar"},
+	{"4e54d2c721cbdb730f01b10b62dec622962b36966ec685880effa63d71c808f2", "foo/../../baz"},
+}
+
+func store(t testing.TB, b backend.Backend, tpe backend.Type, data []byte) {
+	id := backend.Hash(data)
+
+	blob, err := b.Create()
+	OK(t, err)
+
+	_, err = blob.Write([]byte(data))
+	OK(t, err)
+	OK(t, blob.Finalize(tpe, id.String()))
+}
+
+func read(t testing.TB, rd io.Reader, expectedData []byte) {
+	buf, err := ioutil.ReadAll(rd)
+	OK(t, err)
+	if expectedData != nil {
+		Equals(t, expectedData, buf)
+	}
+}
+
+// Generic tests all functions of the backend.
+func Generic(t testing.TB) {
+	b := open(t)
+	defer close(t)
+
+	for _, tpe := range []backend.Type{
+		backend.Data, backend.Key, backend.Lock,
+		backend.Snapshot, backend.Index,
+	} {
+		// detect non-existing files
+		for _, test := range testStrings {
+			id, err := backend.ParseID(test.id)
+			OK(t, err)
+
+			// test if blob is already in repository
+			ret, err := b.Test(tpe, id.String())
+			OK(t, err)
+			Assert(t, !ret, "blob was found to exist before creating")
+
+			// try to open not existing blob
+			_, err = b.GetReader(tpe, id.String(), 0, 0)
+			Assert(t, err != nil, "blob data could be extracted before creation")
+
+			// try to read not existing blob
+			_, err = b.GetReader(tpe, id.String(), 0, 1)
+			Assert(t, err != nil, "blob reader could be obtained before creation")
+
+			// try to get string out, should fail
+			ret, err = b.Test(tpe, id.String())
+			OK(t, err)
+			Assert(t, !ret, "id %q was found (but should not have)", test.id)
+		}
+
+		// add files
+		for _, test := range testStrings {
+			store(t, b, tpe, []byte(test.data))
+
+			// test GetReader()
+			rd, err := b.GetReader(tpe, test.id, 0, uint(len(test.data)))
+			OK(t, err)
+			Assert(t, rd != nil, "GetReader() returned nil")
+
+			read(t, rd, []byte(test.data))
+			OK(t, rd.Close())
+
+			// try to read it out with an offset and a length
+			start := 1
+			end := len(test.data) - 2
+			length := end - start
+			rd, err = b.GetReader(tpe, test.id, uint(start), uint(length))
+			OK(t, err)
+			Assert(t, rd != nil, "GetReader() returned nil")
+
+			read(t, rd, []byte(test.data[start:end]))
+			OK(t, rd.Close())
+		}
+
+		// test adding the first file again
+		test := testStrings[0]
+
+		// create blob
+		blob, err := b.Create()
+		OK(t, err)
+
+		_, err = blob.Write([]byte(test.data))
+		OK(t, err)
+		err = blob.Finalize(tpe, test.id)
+		Assert(t, err != nil, "expected error, got %v", err)
+
+		// remove and recreate
+		err = b.Remove(tpe, test.id)
+		OK(t, err)
+
+		// test that the blob is gone
+		ok, err := b.Test(tpe, test.id)
+		OK(t, err)
+		Assert(t, ok == false, "removed blob still present")
+
+		// create blob
+		blob, err = b.Create()
+		OK(t, err)
+
+		_, err = io.Copy(blob, bytes.NewReader([]byte(test.data)))
+		OK(t, err)
+		OK(t, blob.Finalize(tpe, test.id))
+
+		// list items
+		IDs := backend.IDs{}
+
+		for _, test := range testStrings {
+			id, err := backend.ParseID(test.id)
+			OK(t, err)
+			IDs = append(IDs, id)
+		}
+
+		list := backend.IDs{}
+
+		for s := range b.List(tpe, nil) {
+			list = append(list, ParseID(s))
+		}
+
+		if len(IDs) != len(list) {
+			t.Fatalf("wrong number of IDs returned: want %d, got %d", len(IDs), len(list))
+		}
+
+		sort.Sort(IDs)
+		sort.Sort(list)
+
+		if !reflect.DeepEqual(IDs, list) {
+			t.Fatalf("lists aren't equal, want:\n  %v\n  got:\n%v\n", IDs, list)
+		}
+
+		// remove content if requested
+		if TestCleanup {
+			for _, test := range testStrings {
+				id, err := backend.ParseID(test.id)
+				OK(t, err)
+
+				found, err := b.Test(tpe, id.String())
+				OK(t, err)
+
+				OK(t, b.Remove(tpe, id.String()))
+
+				found, err = b.Test(tpe, id.String())
+				OK(t, err)
+				Assert(t, !found, fmt.Sprintf("id %q not found after removal", id))
+			}
+		}
+	}
+}
+
+// Delete tests the Delete function.
+func Delete(t testing.TB) {
+	b := open(t)
+	defer close(t)
+
+	be, ok := b.(backend.Deleter)
+	if !ok {
+		return
+	}
+
+	err := be.Delete()
+	if err != nil {
+		t.Fatalf("error deleting backend: %v", err)
+	}
+}
+
+// Cleanup runs the cleanup function after all tests are run.
+func Cleanup(t testing.TB) {
+	if CleanupFn == nil {
+		t.Log("CleanupFn function not set")
+		return
+	}
+
+	if !TestCleanup {
+		t.Logf("not cleaning up backend")
+		return
+	}
+
+	err := CleanupFn()
+	if err != nil {
+		t.Fatalf("Cleanup returned error: %v", err)
+	}
+}

backend/test/tests_test.go

@@ -0,0 +1,3 @@
+package test
+
+// func Test