Add last integration tests, remove testsuite

archiver.go

@@ -534,7 +534,7 @@ func (j archiveJob) Copy() pipe.Job {
 func (arch *Archiver) Snapshot(p *Progress, paths []string, parentID backend.ID) (*Snapshot, backend.ID, error) {
 	debug.Log("Archiver.Snapshot", "start for %v", paths)
 
-	debug.Break("Archiver.Snapshot")
+	debug.RunHook("Archiver.Snapshot", nil)
 	sort.Strings(paths)
 
 	// signal the whole pipeline to stop

cmd/restic/integration_test.go

@@ -14,6 +14,7 @@ import (
 	"testing"
 
 	"github.com/restic/restic/backend"
+	"github.com/restic/restic/debug"
 	. "github.com/restic/restic/test"
 )
 
@@ -175,6 +176,80 @@ func TestBackupNonExistingFile(t *testing.T) {
 	})
 }
 
+func TestBackupMissingFile1(t *testing.T) {
+	withTestEnvironment(t, func(env *testEnvironment, global GlobalOptions) {
+		datafile := filepath.Join("testdata", "backup-data.tar.gz")
+		fd, err := os.Open(datafile)
+		if os.IsNotExist(err) {
+			t.Skipf("unable to find data file %q, skipping", datafile)
+			return
+		}
+		OK(t, err)
+		OK(t, fd.Close())
+
+		setupTarTestFixture(t, env.testdata, datafile)
+
+		cmdInit(t, global)
+
+		ranHook := false
+		debug.Hook("pipe.walk1", func(context interface{}) {
+			pathname := context.(string)
+
+			if pathname != filepath.Join("testdata", "0", "0", "9") {
+				return
+			}
+
+			t.Logf("in hook, removing test file testdata/0/0/9/37")
+			ranHook = true
+
+			OK(t, os.Remove(filepath.Join(env.testdata, "0", "0", "9", "37")))
+		})
+
+		cmdBackup(t, global, []string{env.testdata}, nil)
+		cmdFsck(t, global)
+
+		Assert(t, ranHook, "hook did not run")
+		debug.RemoveHook("pipe.walk1")
+	})
+}
+
+func TestBackupMissingFile2(t *testing.T) {
+	withTestEnvironment(t, func(env *testEnvironment, global GlobalOptions) {
+		datafile := filepath.Join("testdata", "backup-data.tar.gz")
+		fd, err := os.Open(datafile)
+		if os.IsNotExist(err) {
+			t.Skipf("unable to find data file %q, skipping", datafile)
+			return
+		}
+		OK(t, err)
+		OK(t, fd.Close())
+
+		setupTarTestFixture(t, env.testdata, datafile)
+
+		cmdInit(t, global)
+
+		ranHook := false
+		debug.Hook("pipe.walk2", func(context interface{}) {
+			pathname := context.(string)
+
+			if pathname != filepath.Join("testdata", "0", "0", "9", "37") {
+				return
+			}
+
+			t.Logf("in hook, removing test file testdata/0/0/9/37")
+			ranHook = true
+
+			OK(t, os.Remove(filepath.Join(env.testdata, "0", "0", "9", "37")))
+		})
+
+		cmdBackup(t, global, []string{env.testdata}, nil)
+		cmdFsck(t, global)
+
+		Assert(t, ranHook, "hook did not run")
+		debug.RemoveHook("pipe.walk2")
+	})
+}
+
 const (
 	incrementalFirstWrite  = 20 * 1042 * 1024
 	incrementalSecondWrite = 12 * 1042 * 1024
@@ -205,15 +280,6 @@ func appendRandomData(filename string, bytes uint) error {
 
 func TestIncrementalBackup(t *testing.T) {
 	withTestEnvironment(t, func(env *testEnvironment, global GlobalOptions) {
-		datafile := filepath.Join("testdata", "backup-data.tar.gz")
-		fd, err := os.Open(datafile)
-		if os.IsNotExist(err) {
-			t.Skipf("unable to find data file %q, skipping", datafile)
-			return
-		}
-		OK(t, err)
-		OK(t, fd.Close())
-
 		cmdInit(t, global)
 
 		datadir := filepath.Join(env.base, "testdata")

debug/debug.go

@@ -18,7 +18,6 @@ import (
 var opts struct {
 	logger *log.Logger
 	tags   map[string]bool
-	breaks map[string]bool
 	m      sync.Mutex
 }
 
@@ -29,7 +28,6 @@ var _ = initDebug()
 func initDebug() bool {
 	initDebugLogger()
 	initDebugTags()
-	initDebugBreaks()
 
 	fmt.Fprintf(os.Stderr, "debug enabled\n")
 
@@ -105,25 +103,6 @@ func initDebugTags() {
 	fmt.Fprintf(os.Stderr, "debug log enabled for: %v\n", tags)
 }
 
-func initDebugBreaks() {
-	opts.breaks = make(map[string]bool)
-
-	env := os.Getenv("DEBUG_BREAK")
-	if len(env) == 0 {
-		return
-	}
-
-	breaks := []string{}
-
-	for _, tag := range strings.Split(env, ",") {
-		t := strings.TrimSpace(tag)
-		opts.breaks[t] = true
-		breaks = append(breaks, t)
-	}
-
-	fmt.Fprintf(os.Stderr, "debug breaks enabled for: %v\n", breaks)
-}
-
 // taken from https://github.com/VividCortex/trace
 func goroutineNum() int {
 	b := make([]byte, 20)
@@ -194,38 +173,3 @@ func Log(tag string, f string, args ...interface{}) {
 		dbgprint()
 	}
 }
-
-// Break stops the program if the debug tag is active and the string in tag is
-// contained in the DEBUG_BREAK environment variable.
-func Break(tag string) {
-	// check if breaking is enabled
-	if v, ok := opts.breaks[tag]; !ok || !v {
-		return
-	}
-
-	_, file, line, _ := runtime.Caller(1)
-	Log("break", "stopping process %d at %s (%v:%v)\n", os.Getpid(), tag, file, line)
-	p, err := os.FindProcess(os.Getpid())
-	if err != nil {
-		panic(err)
-	}
-
-	err = p.Signal(syscall.SIGSTOP)
-	if err != nil {
-		panic(err)
-	}
-}
-
-// BreakIf stops the program if the debug tag is active and the string in tag
-// is contained in the DEBUG_BREAK environment variable and the return value of
-// fn is true.
-func BreakIf(tag string, fn func() bool) {
-	// check if breaking is enabled
-	if v, ok := opts.breaks[tag]; !ok || !v {
-		return
-	}
-
-	if fn() {
-		Break(tag)
-	}
-}

debug/debug_release.go

@@ -3,7 +3,3 @@
 package debug
 
 func Log(tag string, fmt string, args ...interface{}) {}
-
-func Break(string) {}
-
-func BreakIf(string, func() bool) {}

debug/hooks.go

@@ -0,0 +1,28 @@
+// +build !release
+
+package debug
+
+var (
+	hooks map[string]func(interface{})
+)
+
+func init() {
+	hooks = make(map[string]func(interface{}))
+}
+
+func Hook(name string, f func(interface{})) {
+	hooks[name] = f
+}
+
+func RunHook(name string, context interface{}) {
+	f, ok := hooks[name]
+	if !ok {
+		return
+	}
+
+	f(context)
+}
+
+func RemoveHook(name string) {
+	delete(hooks, name)
+}

debug/hooks_release.go

@@ -0,0 +1,9 @@
+// +build release
+
+package debug
+
+func Hook(name string, f func(interface{})) {}
+
+func RunHook(name string, context interface{}) {}
+
+func RemoveHook(name string) {}

pipe/pipe.go

@@ -4,7 +4,6 @@ import (
 	"errors"
 	"fmt"
 	"os"
-	"path"
 	"path/filepath"
 	"sort"
 
@@ -108,17 +107,7 @@ func walk(basedir, dir string, done chan struct{}, jobs chan<- Job, res chan<- R
 
 	// Insert breakpoint to allow testing behaviour with vanishing files
 	// between Readdir() and lstat()
-	debug.BreakIf("pipe.walk1", func() bool {
-		match, err := path.Match(os.Getenv("DEBUG_BREAK_PIPE"), relpath)
-		if err != nil {
-			panic(err)
-		}
-		if match {
-			debug.Log("break", "break pattern matches for %v\n", relpath)
-		}
-
-		return match
-	})
+	debug.RunHook("pipe.walk1", relpath)
 
 	entries := make([]<-chan Result, 0, len(names))
 
@@ -140,19 +129,7 @@ func walk(basedir, dir string, done chan struct{}, jobs chan<- Job, res chan<- R
 
 		// Insert breakpoint to allow testing behaviour with vanishing files
 		// between walk and open
-		debug.BreakIf("pipe.walk2", func() bool {
-			p := filepath.Join(relpath, name)
-
-			match, err := path.Match(os.Getenv("DEBUG_BREAK_PIPE"), p)
-			if err != nil {
-				panic(err)
-			}
-			if match {
-				debug.Log("break", "break pattern matches for %v\n", p)
-			}
-
-			return match
-		})
+		debug.RunHook("pipe.walk2", filepath.Join(relpath, name))
 
 		if isDir(fi) {
 			err = walk(basedir, subpath, done, jobs, ch)

testsuite.sh

@@ -1,12 +0,0 @@
-#!/bin/bash
-
-# tempdir for binaries
-export BASEDIR="$(mktemp --tmpdir --directory restic-testsuite-XXXXXX)"
-export DEBUG_LOG="${BASEDIR}/restic.log"
-
-export TZ=UTC
-
-echo "restic testsuite basedir ${BASEDIR}"
-
-# run tests
-testsuite/run.sh "$@"

testsuite/run.sh

@@ -1,117 +0,0 @@
-#!/bin/bash
-
-set -e
-
-export dir=$(dirname "$0")
-export fake_data_file="${dir}/fake-data.tar.gz"
-
-prepare() {
-    export BASE="$(mktemp --tmpdir --directory restic-testsuite-XXXXXX)"
-    export RESTIC_REPOSITORY="${BASE}/restic-backup"
-    export RESTIC_PASSWORD="foobar"
-    export DATADIR="${BASE}/fake-data"
-    debug "repository is at ${RESTIC_REPOSITORY}"
-
-    mkdir -p "$DATADIR"
-    (cd "$DATADIR"; tar xz) < "$fake_data_file"
-    debug "extracted fake data to ${DATADIR}"
-}
-
-cleanup() {
-    if [ "$DEBUG" = "1" ]; then
-        debug "leaving dir ${BASE}"
-        return
-    fi
-
-    rm -rf "${BASE}"
-    debug "removed dir ${BASE}"
-    unset BASE
-    unset RESTIC_REPOSITORY
-}
-
-msg() {
-    printf "%s\n" "$*"
-}
-
-pass() {
-    printf "\e[32m%s\e[39m\n" "$*"
-}
-
-err() {
-    printf "\e[31m%s\e[39m\n" "$*"
-}
-
-debug() {
-    if [ "$DEBUG" = "1" ]; then
-        printf "\e[33m%s\e[39m\n" "$*"
-    fi
-}
-
-fail() {
-    err "$@"
-    exit 1
-}
-
-run() {
-    if [ "$DEBUG" = "1" ]; then
-        "$@"
-    else
-        "$@" > /dev/null
-    fi
-}
-
-export -f prepare cleanup msg debug pass err fail run
-
-if [ -z "$BASEDIR" ]; then
-    echo "BASEDIR not set" >&2
-    exit 2
-fi
-
-which restic > /dev/null || fail "restic binary not found!"
-which restic.debug > /dev/null || fail "restic.debug binary not found!"
-which dirdiff > /dev/null || fail "dirdiff binary not found!"
-
-debug "restic path: $(which restic)"
-debug "restic.debug path: $(which restic.debug)"
-debug "dirdiff path: $(which dirdiff)"
-debug "path: $PATH"
-
-debug "restic versions:"
-run restic version
-run restic.debug version
-
-if [ "$#" -gt 0 ]; then
-    testfiles="$1"
-else
-    testfiles=(${dir}/test-*.sh)
-fi
-
-echo "testfiles: ${testfiles[@]}"
-
-failed=""
-for testfile in "${testfiles[@]}"; do
-    msg "================================================================================"
-    msg "run test $testfile"
-    msg ""
-
-    current=$(basename "${testfile}" .sh)
-
-    if [ "$DEBUG" = "1" ]; then
-        OPTS="-v"
-    fi
-
-    if bash $OPTS "${testfile}"; then
-        pass "${current} pass"
-    else
-        err "${current} failed!"
-        failed+=" ${current}"
-    fi
-done
-
-if [ -n "$failed" ]; then
-    err "failed tests: ${failed}"
-    msg "restic versions:"
-    run restic version
-    run restic.debug version
-    exit 1
-fi

testsuite/test-backup-missing-file1.sh

@@ -1,16 +0,0 @@
-set -em
-
-# setup restic
-prepare
-run restic init
-
-# start backup, break between readdir and lstat
-DEBUG_BREAK=pipe.walk1 DEBUG_BREAK_PIPE="fake-data/0/0/9" run restic.debug backup "${BASE}/fake-data" && debug "done"
-
-# remove file
-rm -f "${BASE}/fake-data/0/0/9/37"
-
-# resume backup
-fg
-
-cleanup

testsuite/test-backup-missing-file2.sh

@@ -1,16 +0,0 @@
-set -em
-
-# setup restic
-prepare
-run restic init
-
-# start backup, break between walk and save
-DEBUG_BREAK=pipe.walk2 DEBUG_BREAK_PIPE="fake-data/0/0/9/37" run restic.debug backup "${BASE}/fake-data" && debug "done"
-
-# remove file
-rm -f "${BASE}/fake-data/0/0/9/37"
-
-# resume backup
-fg
-
-cleanup